diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 0ebe8ccae5a8c46cc2aee5c339ce9186410691b4..e8ecd18336fc2faab7236f714e331869dad484cb 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -675,21 +675,13 @@ static bool stage2_has_fwb(struct kvm_pgtable *pgt) void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, size_t size) { - unsigned long pages, inval_pages; + unsigned long pages = size >> PAGE_SHIFT; - if (!system_supports_tlb_range()) { + if (!system_supports_tlb_range() || pages > MAX_TLBI_RANGE_PAGES) { kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); return; } - - pages = size >> PAGE_SHIFT; - while (pages > 0) { - inval_pages = min(pages, MAX_TLBI_RANGE_PAGES); - kvm_call_hyp(__kvm_tlb_flush_vmid_range, mmu, addr, inval_pages); - - addr += inval_pages << PAGE_SHIFT; - pages -= inval_pages; - } + kvm_call_hyp(__kvm_tlb_flush_vmid_range, mmu, addr, pages); } #define KVM_S2_MEMATTR(pgt, attr) PAGE_S2_MEMATTR(attr, stage2_has_fwb(pgt)) diff --git a/drivers/firmware/uvb/Makefile b/drivers/firmware/uvb/Makefile index d5a261794d9e4d909a5adf8da6281fd94d4d493e..ae66ef7e511cb686c7f68844a52477e887eb55e1 100644 --- a/drivers/firmware/uvb/Makefile +++ b/drivers/firmware/uvb/Makefile @@ -13,4 +13,5 @@ odf-objs := odf/odf_trans.o \ obj-$(CONFIG_UDFI_CIS) += cis.o cis-objs := cis/cis_info_process.o \ cis/uvb_info_process.o \ - cis/cis_core.o + cis/cis_core.o \ + cis/cis_ub.o diff --git a/drivers/firmware/uvb/cis/cis_core.c b/drivers/firmware/uvb/cis/cis_core.c index 4aa2d858ac6708e2aee2f7c4ff6eb4f9398ce65d..ee58031283bd10258569e82ab6a438835965082f 100644 --- a/drivers/firmware/uvb/cis/cis_core.c +++ b/drivers/firmware/uvb/cis/cis_core.c @@ -13,8 +13,11 @@ #include #include #include +#include +#include #include "cis_info_process.h" #include "uvb_info_process.h" +#include "cis_ub.h" MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Call ID Service Framework"); @@ -73,7 +76,7 @@ static int uvb_window_lock_init(void) free_uvb_window_lock(); return -ENOMEM; } - lock_node->lock.counter = 0; + atomic_set(&lock_node->lock, 0); lock_node->window_address = uvb->wd[j].address; hash_add(uvb_lock_table, &lock_node->node, uvb->wd[j].address); } @@ -140,6 +143,22 @@ void uninit_uvb(void) free_uvb_window_lock(); } +static const struct auxiliary_device_id uvb_id_table[] = { + { + .name = UBASE_ADEV_NAME ".uvb", + }, + {}, +}; + +MODULE_DEVICE_TABLE(auxiliary, uvb_id_table); + +static struct auxiliary_driver uvb_drv = { + .probe = uvb_probe, + .remove = uvb_remove, + .name = "uvb", + .id_table = uvb_id_table, +}; + static int __init cis_init(void) { int err = 0; @@ -147,23 +166,29 @@ static int __init cis_init(void) err = init_cis_table(); if (err) { pr_err("cis info init failed, err=%d\n", err); - return err; - } + } else { + err = init_global_vars(); + if (err) { + pr_err("global vars malloc failed, err=%d\n", err); + return err; + } - err = init_global_vars(); - if (err) { - pr_err("global vars malloc failed, err=%d\n", err); - return err; + err = init_uvb(); + if (err) { + pr_err("uvb init failed, err=%d\n", err); + free_global_vars(); + return err; + } + pr_info("cis uvb init success\n"); } - err = init_uvb(); + err = auxiliary_driver_register(&uvb_drv); if (err) { - pr_err("uvb init failed, err=%d\n", err); - free_global_vars(); + pr_err("failed to register uvb drv\n"); return err; } - pr_info("cis init success\n"); + pr_info("register uvb over ub drv success\n"); return 0; } @@ -172,6 +197,7 @@ static void __exit cis_exit(void) { uninit_uvb(); free_global_vars(); + auxiliary_driver_unregister(&uvb_drv); pr_info("cis exit success\n"); } diff --git a/drivers/firmware/uvb/cis/cis_info_process.c b/drivers/firmware/uvb/cis/cis_info_process.c index d0811b2bd9fe59a999fcdac3b07bdf5adf51cfd5..561a7b3844816023c3508019af5ea2e2f73ead98 100644 --- a/drivers/firmware/uvb/cis/cis_info_process.c +++ b/drivers/firmware/uvb/cis/cis_info_process.c @@ -165,7 +165,7 @@ static int try_obtain_uvb_window(u64 *wd_obtain, u32 sender_id) return 0; } -struct uvb_window_description *uvb_occupy_window(struct uvb *uvb, u32 sender_id, u64 *wd_obtain) +struct uvb_window_description *uvb_occupy_window(struct uvb *uvb, u32 sender_id, u64 **wd_obtain) { struct uvb_window_description *wd = NULL; ktime_t start; @@ -184,8 +184,8 @@ struct uvb_window_description *uvb_occupy_window(struct uvb *uvb, u32 sender_id, round++; } wd = &(uvb->wd[i]); - wd_obtain = memremap(wd->obtain, wd->size, MEMREMAP_WC); - if (!wd_obtain) { + *wd_obtain = memremap(wd->obtain, sizeof(u64), MEMREMAP_WC); + if (!*wd_obtain) { pr_err("uvb window obtain map failed\n"); return NULL; } @@ -195,17 +195,16 @@ struct uvb_window_description *uvb_occupy_window(struct uvb *uvb, u32 sender_id, goto free_resources; } - if (atomic_cmpxchg(lock, 0, 1) == 0 - && try_obtain_uvb_window(wd_obtain, sender_id)) { - atomic_set(lock, 0); - udelay(uvb->delay); - if (*wd_obtain == sender_id) { - now = ktime_get(); - time_interval = ktime_to_us(ktime_sub(now, start)); - pr_info("occupy uvb window successfully, elapsed time: %lldus\n", - time_interval); - return wd; + if (atomic_cmpxchg(lock, 0, 1) == 0) { + if (try_obtain_uvb_window(*wd_obtain, sender_id)) { + udelay(uvb->delay); + if (**wd_obtain == sender_id) { + atomic_set(lock, 0); + pr_info("occupy uvb window successfully\n"); + return wd; + } } + atomic_set(lock, 0); } now = ktime_get(); @@ -216,13 +215,13 @@ struct uvb_window_description *uvb_occupy_window(struct uvb *uvb, u32 sender_id, goto free_resources; } i++; - memunmap(wd_obtain); - wd_obtain = NULL; + memunmap(*wd_obtain); + *wd_obtain = NULL; } free_resources: - memunmap(wd_obtain); - wd_obtain = NULL; + memunmap(*wd_obtain); + *wd_obtain = NULL; return NULL; } @@ -231,7 +230,8 @@ void uvb_free_wd_obtain(u64 *wd_obtain) { if (!wd_obtain) return; - *wd_obtain = 0; + if (*wd_obtain) + *wd_obtain = 0; memunmap(wd_obtain); } @@ -406,7 +406,7 @@ void free_io_param_with_buffer(struct cis_message *io_param) int cis_call_uvb(u8 index, struct udfi_para *para) { - int err; + int err = 0; struct uvb_window *window = NULL; struct uvb_window_description *wd = NULL; struct cis_message *io_param = NULL; @@ -422,7 +422,7 @@ int cis_call_uvb(u8 index, struct udfi_para *para) return -EOVERFLOW; } - wd = uvb_occupy_window(g_uvb_info->uvbs[index], para->sender_id, wd_obtain); + wd = uvb_occupy_window(g_uvb_info->uvbs[index], para->sender_id, &wd_obtain); if (!wd) { pr_err("obtain window failed\n"); return -EBUSY; @@ -430,6 +430,7 @@ int cis_call_uvb(u8 index, struct udfi_para *para) if (!wd->buffer) { pr_err("no window buffer to save data\n"); + err = -EOPNOTSUPP; goto free_obtain; } @@ -477,7 +478,7 @@ int cis_call_uvb(u8 index, struct udfi_para *para) int cis_call_uvb_sync(u8 index, struct udfi_para *para) { - int err; + int err = 0; struct uvb_window *window = NULL; struct uvb_window_description *wd = NULL; u64 *wd_obtain = NULL; @@ -494,7 +495,7 @@ int cis_call_uvb_sync(u8 index, struct udfi_para *para) return -EOVERFLOW; } - wd = uvb_occupy_window(g_uvb_info->uvbs[index], para->sender_id, wd_obtain); + wd = uvb_occupy_window(g_uvb_info->uvbs[index], para->sender_id, &wd_obtain); if (!wd) { pr_err("sync call obtain window failed\n"); return -EBUSY; @@ -502,6 +503,7 @@ int cis_call_uvb_sync(u8 index, struct udfi_para *para) if (!wd->buffer) { pr_err("sync call no window buffer to save data\n"); + err = -EOPNOTSUPP; goto free_obtain; } @@ -685,6 +687,7 @@ int register_local_cis_func(u32 call_id, u32 receiver_id, msg_handler func) if (!p) return -ENOMEM; + INIT_LIST_HEAD(&p->link); p->call_id = call_id; p->receiver_id = receiver_id; p->func = func; @@ -692,7 +695,7 @@ int register_local_cis_func(u32 call_id, u32 receiver_id, msg_handler func) spin_lock_irqsave(&cis_register_lock, flags); list_add_tail_rcu(&p->link, &g_local_cis_list); spin_unlock_irqrestore(&cis_register_lock, flags); - pr_info("register cis func success\n"); + pr_info("register cis func for callid=%08x recvid=%08x success\n", call_id, receiver_id); return 0; } @@ -726,7 +729,8 @@ int unregister_local_cis_func(u32 call_id, u32 receiver_id) synchronize_rcu(); kfree(p); - pr_info("unregister cis func success\n"); + pr_info("unregister cis func for callid=%08x recvid=%08x success\n", + call_id, receiver_id); return 0; } diff --git a/drivers/firmware/uvb/cis/cis_info_process.h b/drivers/firmware/uvb/cis/cis_info_process.h index 4713e5306d7714ed7aa0e585a9fb666d97af77ec..c28e4bbaa168c3cd1916500aa29e13f7e6319761 100644 --- a/drivers/firmware/uvb/cis/cis_info_process.h +++ b/drivers/firmware/uvb/cis/cis_info_process.h @@ -21,7 +21,6 @@ extern struct cis_message *io_param_sync; extern struct list_head g_local_cis_list; -extern DECLARE_HASHTABLE(uvb_lock_table, MAX_UVB_LOCK_IN_BITS); struct udfi_para { u32 message_id; diff --git a/drivers/firmware/uvb/cis/cis_ub.c b/drivers/firmware/uvb/cis/cis_ub.c new file mode 100644 index 0000000000000000000000000000000000000000..37183c7c8d4fe2966796dd78f0c726b57796ff10 --- /dev/null +++ b/drivers/firmware/uvb/cis/cis_ub.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + * Description: UVB over UB message processing module, handles CIS UB init, + * func register/lookup and UB message. + * Author: mengkanglai + * Create: 2025-12-18 + */ + +#define pr_fmt(fmt) "[UVB_UB]: " fmt + +#include +#include +#include +#include +#include +#include "cis_ub.h" +#include "cis_info_process.h" + +enum uvb_opcode { + CIS_OPC_SEND = 0xFA00, + CIS_OPC_RECV = 0xFA01, +}; + +void ub_msg_header_init(struct msg_pkt_header *header) +{ + header->ulh.cfg = UB_COMPACT_LINK_CFG; + header->nth.nth_nlp = NTH_NLP_WITH_TPH; + header->nth.mm = NTH_NLP_WITH_MGMT; + header->ctph_nlp = CTPH_NLP_UPI_40BITS_UEID; + header->tp_opcode = CTPH_OPCODE_NOT_CNP; + header->btah.ta_opcode = TAH_OPCODE_MSG; + header->msgetah.msg_code = UB_MSG_CODE; + header->msgetah.sub_msg_code = UB_SUB_MSG_CODE; + header->pload.type = UB_PAYLOAD_TYPE; + header->pload.version = UB_PAYLOAD_VERSION; +} + +void setup_resp_pkt(struct msg_pkt_header *resp, struct msg_pkt_header *rcv) +{ + struct compact_network_header *nth = &rcv->nth; + + ub_msg_header_init(resp); + resp->upi = rcv->upi; + resp->pload.message_id = ~rcv->pload.message_id; + resp->pload.sender_id = rcv->pload.sender_id; + resp->pload.receiver_id = rcv->pload.receiver_id; + resp->nth.dcna = nth->scna; + resp->deid = eid_gen(rcv->seid_h, rcv->seid_l); + resp->nth.scna = nth->dcna; + resp->seid_l = seid_low(rcv->deid); + resp->seid_h = seid_high(rcv->deid); + resp->msgetah.rsp = 0x1; + resp->btah.ini_tassn = rcv->btah.ini_tassn; + resp->btah.ini_rc_id = rcv->btah.ini_rc_id; + resp->btah.ee = rcv->btah.ee; +} + +static bool check_pload_valid(struct payload *pload, u32 len) +{ + if (!pload->sender_id || !pload->receiver_id) { + pr_err("senderid or receiverid can't be null\n"); + return false; + } + + if (UBIOS_GET_MESSAGE_FLAG(pload->message_id) != UBIOS_CALL_ID_FLAG) { + pr_err("uvb over ub not a cis call\n"); + return false; + } + + if (len > sizeof(struct msg_pkt_header) && !pload->data_total_size) { + pr_err("ub pload have data but data size is null\n"); + return false; + } + + if (len == sizeof(struct msg_pkt_header) && pload->data_total_size) { + pr_err("ub pload no data but data size is not null\n"); + return false; + } + + return true; +} + +int uvb_handle_cmdq_event(void *dev, void *data, u32 len) +{ + int err = 0; + msg_handler func; + struct cis_message msg = {}; + struct ubase_cmd_buf in; + struct msg_pkt_header *resp; + struct msg_pkt_header *ub_header = (struct msg_pkt_header *)data; + struct payload *pload = &ub_header->pload; + u32 output_size = pload->output_buffer_size == UVB_OUTPUT_SIZE_NULL ? 0 : + pload->output_buffer_size; + msg.input = pload->data; + msg.input_size = pload->data_total_size; + + if (ub_header->msgetah.plen - sizeof(struct payload) < pload->data_total_size) { + pr_err("it's a framed message, not supported\n"); + return -EINVAL; + } + + if (!check_pload_valid(pload, len)) { + pr_err("check payload param failed\n"); + return -EINVAL; + } + + pr_info("handle msg for callid=%08x, receverid=%08x\n", + pload->message_id, pload->receiver_id); + func = search_local_cis_func(pload->message_id, pload->receiver_id); + if (!func) { + pr_err("not found registered func\n"); + return -EOPNOTSUPP; + } + + resp = kzalloc(sizeof(struct msg_pkt_header) + output_size, GFP_KERNEL); + if (!resp) + return -ENOMEM; + + if (!output_size) { + msg.p_output_size = NULL; + msg.output = NULL; + } else { + resp->pload.data_total_size = output_size; + msg.p_output_size = &resp->pload.data_total_size; + msg.output = resp->pload.data; + } + + err = func(&msg); + if (err) + pr_err("execute cis func failed for %08x, err=%d\n", pload->message_id, err); + + setup_resp_pkt(resp, ub_header); + resp->msgetah.plen = resp->pload.data_total_size + sizeof(struct payload); + resp->pload.returned_status = (u32)err; + + ubase_fill_inout_buf(&in, CIS_OPC_SEND, false, + sizeof(struct msg_pkt_header) + resp->pload.data_total_size, resp); + err = ubase_cmd_send_in(dev, &in); + if (err) + pr_err("send ubase info err, err=%d\n", err); + + kfree(resp); + + return err; +} + +struct ubase_crq_event_nb uvb_crq_events = { + .opcode = CIS_OPC_RECV, + .crq_handler = uvb_handle_cmdq_event, +}; + +int uvb_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id) +{ + int ret = 0; + + uvb_crq_events.back = aux_dev; + ret = ubase_register_crq_event(aux_dev, &uvb_crq_events); + if (ret) { + pr_err("register uvb crq events failed, err=%d\n", ret); + return -1; + } + + pr_info("register uvb crq events success\n"); + + return 0; +} + +void uvb_remove(struct auxiliary_device *aux_dev) +{ + ubase_unregister_crq_event(aux_dev, CIS_OPC_RECV); +} diff --git a/drivers/firmware/uvb/cis/cis_ub.h b/drivers/firmware/uvb/cis/cis_ub.h new file mode 100644 index 0000000000000000000000000000000000000000..c8a31d0dc1e97e330215da56edff4b156142a39d --- /dev/null +++ b/drivers/firmware/uvb/cis/cis_ub.h @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + * Description: cis ub header + * Create: 2025-12-18 + */ + +#ifndef CIS_UB_H +#define CIS_UB_H + +#define seid_high(eid) (((eid) >> 12) & 0xff) +#define seid_low(eid) ((eid) & 0xfff) +#define eid_gen(eid_h, eid_l) ((eid_h) << 12 | (eid_l)) + +struct payload { +#define UB_PAYLOAD_TYPE 0x1 + u32 type : 4; +#define UB_PAYLOAD_VERSION 0x1 + u32 version : 4; + u32 rsvd : 8; + u32 index : 16; + u32 message_id; + u32 sender_id; + u32 receiver_id; + union { + u32 output_buffer_size; + u32 returned_status; + }; + u32 data_total_size; + DECLARE_FLEX_ARRAY(char, data); +}; + +struct base_header { + /* DW0 */ + u32 ini_tassn : 16; + u32 udf_flag : 1; + u32 rsvd2 : 1; + u32 poison : 1; + u32 tk_en : 1; + u32 ee : 2; + u32 taver : 2; +#define TAH_OPCODE_MSG 0x10 + u32 ta_opcode : 8; + /* DW1 */ + u32 ini_rc_id : 20; + u32 ini_rc_type : 2; + u32 excl : 1; + u32 rsvd3 : 1; + u32 atloc : 1; + u32 retry : 1; + u32 se : 1; + u32 tc_en : 1; + u32 tak_en : 1; + u32 odr : 3; +}; + +struct msg_extended_header { + u32 plen : 12; + u32 rsvd : 4; + u32 rsp_status : 8; + u32 rsp : 1; +#define UB_MSG_CODE 0x3 + u32 msg_code : 3; +#define UB_SUB_MSG_CODE 0x1 + u32 sub_msg_code : 4; +}; + +struct compact_network_header { + /* DW0 */ + u32 dcna : 16; + u32 scna : 16; + /* DW1 */ +#define NTH_NLP_WITH_TPH 0 +#define NTH_NLP_WITHOUT_TPH 1 + u32 nth_nlp : 3; +#define NTH_NLP_WITH_MGMT 0 + u32 mm : 1; + u32 sl : 4; + u32 lb : 8; + u32 cci : 16; +}; + +struct ub_link_header { + u32 plen : 14; + u32 rt : 2; + u32 cfg : 4; + u32 rsvd0 : 1; + u32 vl : 4; + u32 rsvd1 : 1; + u32 crd_vl : 4; + u32 ack : 1; + u32 crd : 1; +}; + +#define UB_COMPACT_LINK_CFG 6 + +struct msg_pkt_header { + /* DW0 */ + struct ub_link_header ulh; + /* DW1~DW2 */ +struct compact_network_header nth; + /* DW3 */ + u32 seid_h : 8; + u32 upi : 16; +#define CTPH_NLP_UPI_40BITS_UEID 2 + u32 ctph_nlp : 4; + u32 pad : 2; +#define CTPH_OPCODE_NOT_CNP 0 + u32 tp_opcode : 2; + /* DW4 */ + u32 deid : 20; + u32 seid_l : 12; + /* DW5~DW6 */ + struct base_header btah; + /* DW7 */ + struct msg_extended_header msgetah; + + /* DW8~DW15 */ + struct payload pload; +}; + +int uvb_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id); +void uvb_remove(struct auxiliary_device *aux_dev); + +#endif diff --git a/drivers/firmware/uvb/cis/uvb_info_process.h b/drivers/firmware/uvb/cis/uvb_info_process.h index 7f7f0e7362dbf518714557a9cfb28d8c5a487df6..1341ec85c132f81c0255d5442f6df5d800234ead 100644 --- a/drivers/firmware/uvb/cis/uvb_info_process.h +++ b/drivers/firmware/uvb/cis/uvb_info_process.h @@ -9,6 +9,8 @@ #ifndef UVB_INFO_PROCESS_H #define UVB_INFO_PROCESS_H +extern DECLARE_HASHTABLE(uvb_lock_table, MAX_UVB_LOCK_IN_BITS); + int uvb_poll_window(void *data); u32 checksum32(const void *data, u32 size); #endif diff --git a/drivers/firmware/uvb/odf/odf_helper.c b/drivers/firmware/uvb/odf/odf_helper.c index c170d80c5018eda3850e746b959c7da1641f1075..6a50635b1a41ef9935beb1fb4eb645334e1a3a6c 100644 --- a/drivers/firmware/uvb/odf/odf_helper.c +++ b/drivers/firmware/uvb/odf/odf_helper.c @@ -160,7 +160,8 @@ int odf_separate_name(char **path, char *name, u64 max_len, u16 *index) c++; } - if ((i > max_len) || (j >= UBIOS_OD_INDEX_STRING_MAX)) + if ((i == max_len && name[i-1] != '\0') || + (j == UBIOS_OD_INDEX_STRING_MAX && index_string[j-1] != '\0')) return -EOVERFLOW; if (*c == '\0') diff --git a/drivers/firmware/uvb/odf/odf_trans.c b/drivers/firmware/uvb/odf/odf_trans.c index ac68c736db2055630e26926d1c1c464a0cd1377e..ad970b83e4849a76b73c0148ddaec77ec472b695 100644 --- a/drivers/firmware/uvb/odf/odf_trans.c +++ b/drivers/firmware/uvb/odf/odf_trans.c @@ -454,7 +454,7 @@ static int __init odf_init(void) status = create_odf_info(); if (status) { pr_err("odf table init failed\n"); - return -1; + return 0; } status = create_cis_info_from_odf(); @@ -478,7 +478,7 @@ static int __init odf_init(void) free_odf: free_odf_info(); - return -1; + return 0; } static void __exit odf_exit(void) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index dda6115f7d08dc80e747c5b2ce6e0cdf6ddd1257..8758e2f83d6f2937f84c697118c34f47798ab020 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3360,7 +3360,6 @@ static struct iommu_domain arm_smmu_blocked_domain = { static struct iommu_domain * arm_smmu_domain_alloc_user(struct device *dev, u32 flags, struct iommu_domain *parent, - struct kvm *kvm, const struct iommu_user_data *user_data) { struct arm_smmu_master *master = dev_iommu_priv_get(dev); @@ -4003,7 +4002,7 @@ static struct iommu_ops arm_smmu_ops = { .hw_info = arm_smmu_hw_info, .domain_alloc_paging = arm_smmu_domain_alloc_paging, .domain_alloc_sva = arm_smmu_sva_domain_alloc, - .domain_alloc_user_v2 = arm_smmu_domain_alloc_user, + .domain_alloc_user = arm_smmu_domain_alloc_user, .probe_device = arm_smmu_probe_device, .release_device = arm_smmu_release_device, .device_group = arm_smmu_device_group, diff --git a/drivers/ub/obmm/Kconfig b/drivers/ub/obmm/Kconfig index 85d2eff555e16a3c4c4b616dcfb3185d73398fe7..5b88e499cc5c46467fa36d770e8a733e1b40d0c8 100644 --- a/drivers/ub/obmm/Kconfig +++ b/drivers/ub/obmm/Kconfig @@ -2,7 +2,7 @@ config OBMM default n tristate "OBMM(ownership based memory management) Support" - depends on UB_UMMU_CORE && UB_UBUS && HISI_SOC_CACHE + depends on UB_UMMU_CORE && UB_UBUS && HISI_SOC_CACHE && ARCH_HAS_COPY_MC select NUMA_REMOTE select PFN_RANGE_ALLOC select RECLAIM_NOTIFY diff --git a/drivers/ub/obmm/conti_mem_allocator.c b/drivers/ub/obmm/conti_mem_allocator.c index 8ca482f5dfa5fbc8939c8c942d690028d0bde979..17d7c4cb39c5eb1a2e8bc537f1f2215d2fe1b11f 100644 --- a/drivers/ub/obmm/conti_mem_allocator.c +++ b/drivers/ub/obmm/conti_mem_allocator.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "conti_mem_allocator.h" @@ -103,6 +105,7 @@ size_t conti_mem_allocator_expand(struct conti_mem_allocator *allocator, size_t expand_size = size - count * allocator->granu; atomic64_add(expand_size, &allocator->pooled_mem_size); + atomic64_add(expand_size, &allocator->uncleared_mem_size); if (expand_size > 0) pr_debug("%s: expand expect size %#zx, actual size %#zx\n", current->comm, size, expand_size); @@ -116,6 +119,7 @@ size_t conti_mem_allocator_contract(struct conti_mem_allocator *allocator, size_ struct memseg_node *node, *tmp; unsigned long count, flags; size_t contract_size; + size_t from_uncleared = 0, from_ready = 0; if (size == 0 || size % allocator->granu) { pr_err_ratelimited("size %#zx is zero or not aligned with allocator->granu.\n", @@ -129,9 +133,11 @@ size_t conti_mem_allocator_contract(struct conti_mem_allocator *allocator, size_ INIT_LIST_HEAD(&contract_list); spin_lock_irqsave(&allocator->lock, flags); + list_for_each_entry_safe(node, tmp, &allocator->memseg_uncleared, list) { list_move_tail(&node->list, &contract_list); count--; + from_uncleared += allocator->granu; if (count == 0) goto done; } @@ -139,6 +145,7 @@ size_t conti_mem_allocator_contract(struct conti_mem_allocator *allocator, size_ list_for_each_entry_safe(node, tmp, &allocator->memseg_ready, list) { list_move_tail(&node->list, &contract_list); count--; + from_ready += allocator->granu; if (count == 0) goto done; } @@ -151,6 +158,8 @@ size_t conti_mem_allocator_contract(struct conti_mem_allocator *allocator, size_ } contract_size = size - count * allocator->granu; atomic64_sub(contract_size, &allocator->pooled_mem_size); + atomic64_sub(from_uncleared, &allocator->uncleared_mem_size); + atomic64_sub(from_ready, &allocator->ready_mem_size); if (contract_size > 0) pr_debug("%s: nid: %d, contract expect size %#zx, actual size %#zx\n", current->comm, allocator->nid, size, contract_size); @@ -233,7 +242,7 @@ static size_t conti_alloc_memory_slow(struct conti_mem_allocator *allocator, siz atomic64_add(allocated, &allocator->pooled_mem_size); atomic64_add(allocated, &allocator->used_mem_size); - pr_info("%s: slow allocated %#zx from node %d\n", current->comm, allocated, allocator->nid); + pr_debug("%s: slow allocate %#zx from node %d\n", current->comm, allocated, allocator->nid); return allocated; } @@ -242,68 +251,86 @@ size_t conti_alloc_memory(struct conti_mem_allocator *allocator, size_t size, { struct list_head *first, *second, *entry, temp_list; struct memseg_node *node; - size_t allocated = 0, available; + size_t allocated = 0; + size_t first_allocated = 0, second_allocated = 0; unsigned long flags; atomic_inc(&pool_thread_should_pause); INIT_LIST_HEAD(&temp_list); + + /* Determine which lists to allocate from based on clear requirement. + * clear=true: prefer ready list, uncleared requires sync clearing (slow) + * clear=false: prefer uncleared list, ready is also usable + */ if (clear) { first = &allocator->memseg_ready; - second = &allocator->memseg_uncleared; + second = allow_slow ? &allocator->memseg_uncleared : NULL; } else { second = &allocator->memseg_ready; first = &allocator->memseg_uncleared; } spin_lock_irqsave(&allocator->lock, flags); - available = conti_get_avail(allocator); - if (!allow_slow && available < size) { - pr_err("%s:fast alloc failed. nid: %d, request: 0x%lx, available: 0x%lx\n", - __func__, allocator->nid, size, available); - spin_unlock_irqrestore(&allocator->lock, flags); - goto out_continue_pool; - } + + /* Allocate from first list */ list_for_each(entry, first) { if (allocated >= size) break; allocated += allocator->granu; + first_allocated += allocator->granu; pr_debug("alloc 1 node from %s list.\n", clear ? "cleared" : "uncleared"); } list_cut_before(head, first, entry); - list_for_each(entry, second) { - if (allocated >= size) - break; - allocated += allocator->granu; - pr_debug("alloc 1 node from %s list.\n", !clear ? "cleared" : "uncleared"); + /* Allocate from second list if it exists (allow_slow=true for clear=true case) */ + if (second) { + list_for_each(entry, second) { + if (allocated >= size) + break; + allocated += allocator->granu; + second_allocated += allocator->granu; + pr_debug("alloc 1 node from %s list.\n", !clear ? "cleared" : "uncleared"); + } + list_cut_before(&temp_list, second, entry); } - list_cut_before(&temp_list, second, entry); spin_unlock_irqrestore(&allocator->lock, flags); + /* Update statistics counters based on actual list sources */ + if (first == &allocator->memseg_ready) + atomic64_sub(first_allocated, &allocator->ready_mem_size); + else + atomic64_sub(first_allocated, &allocator->uncleared_mem_size); + + if (second) { + if (second == &allocator->memseg_ready) + atomic64_sub(second_allocated, &allocator->ready_mem_size); + else + atomic64_sub(second_allocated, &allocator->uncleared_mem_size); + } atomic64_add(allocated, &allocator->used_mem_size); - /* now: head collects elements from the first list, temp_list holds elements form the - * second list and clearing node. When the caller requests for cleared data, all nodes in - * temp_list should be cleared synchronously. + /* Synchronously clear memory from uncleared list when clear=true. + * head: memory from first list + * temp_list: memory from second list (uncleared), needs clearing */ if (clear) list_for_each_entry(node, &temp_list, list) conti_clear_memseg(allocator, node); list_splice(&temp_list, head); - if (allocated < size) + /* Slow path: allocate new memory via ops when pools are exhausted */ + if (allow_slow && allocated < size) allocated += conti_alloc_memory_slow(allocator, size - allocated, head, clear); list_for_each_entry(node, head, list) { pr_debug("allocator: allocated: %d: 0x%llx + 0x%lx\n", allocator->nid, node->addr, node->size); } - pr_info("%s: allocated %#zx from node %d\n", current->comm, allocated, allocator->nid); + pr_debug("%s: allocated %#zx from node %d\n", current->comm, allocated, allocator->nid); -out_continue_pool: atomic_dec(&pool_thread_should_pause); - /* not aligned */ + /* Sanity check: allocated size should never exceed requested size */ WARN_ON(allocated > size); return allocated; @@ -313,6 +340,7 @@ bool conti_mem_allocator_isolate_memseg(struct conti_mem_allocator *a, unsigned { struct memseg_node *node; bool found = false; + bool from_ready = false; unsigned long flags; if (!a->initialized) @@ -324,6 +352,7 @@ bool conti_mem_allocator_isolate_memseg(struct conti_mem_allocator *a, unsigned pr_debug("isolate memseg from cleared pool.\n"); list_move(&node->list, &a->memseg_poisoned); found = true; + from_ready = true; goto out; } } @@ -342,8 +371,13 @@ bool conti_mem_allocator_isolate_memseg(struct conti_mem_allocator *a, unsigned out: spin_unlock_irqrestore(&a->lock, flags); - if (found) + if (found) { atomic64_sub(a->granu, &a->pooled_mem_size); + if (from_ready) + atomic64_sub(a->granu, &a->ready_mem_size); + else + atomic64_sub(a->granu, &a->uncleared_mem_size); + } return found; } @@ -378,10 +412,15 @@ static int conti_clear_thread(void *p) spin_lock_irqsave(&allocator->lock, flags); allocator->memseg_clearing = NULL; - if (ret) + if (ret) { + /* Clearing failed, return to uncleared */ list_add(&node->list, &allocator->memseg_uncleared); - else + } else { + /* Clearing succeeded: move from uncleared to ready */ + atomic64_sub(allocator->granu, &allocator->uncleared_mem_size); + atomic64_add(allocator->granu, &allocator->ready_mem_size); list_add(&node->list, &allocator->memseg_ready); + } spin_unlock_irqrestore(&allocator->lock, flags); } pr_debug("%s: nid=%d, exit\n", __func__, allocator->nid); @@ -394,7 +433,7 @@ static int clear_thread_init(struct conti_mem_allocator *allocator) struct task_struct *work; work = kthread_create_on_node(conti_clear_thread, allocator, allocator->nid, - "conti_clear_%s", allocator->name); + "clearing-%s", allocator->name); if (IS_ERR(work)) { pr_err("failed to init conti_clear task\n"); return -ENODEV; @@ -464,8 +503,8 @@ static int pool_thread_init(struct conti_mem_allocator *allocator) struct task_struct *work; init_waitqueue_head(&allocator->pool_wq); - work = kthread_create_on_node(conti_pool_thread, allocator, allocator->nid, "conti_pool_%s", - allocator->name); + work = kthread_create_on_node(conti_pool_thread, allocator, allocator->nid, + "pooling-%s", allocator->name); if (IS_ERR(work)) { pr_err("failed to init conti_pool task\n"); return -ENODEV; @@ -477,8 +516,77 @@ static int pool_thread_init(struct conti_mem_allocator *allocator) return 0; } +/* ========== Sysfs support ========== */ + +static void conti_kobj_release(struct kobject *kobj) +{ + /* The kobject is embedded in conti_mem_allocator, nothing to free here */ + /* Memory is managed by the allocator itself */ +} + +static const struct kobj_type conti_ktype = { + .release = conti_kobj_release, + .sysfs_ops = &kobj_sysfs_ops, +}; + +static ssize_t total_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct conti_mem_allocator *allocator; + + allocator = container_of(kobj, struct conti_mem_allocator, kobj); + return sysfs_emit(buf, "0x%llx\n", atomic64_read(&allocator->pooled_mem_size)); +} + +static ssize_t used_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct conti_mem_allocator *allocator; + + allocator = container_of(kobj, struct conti_mem_allocator, kobj); + return sysfs_emit(buf, "0x%llx\n", atomic64_read(&allocator->used_mem_size)); +} + +static ssize_t available_cleared_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct conti_mem_allocator *allocator; + + allocator = container_of(kobj, struct conti_mem_allocator, kobj); + return sysfs_emit(buf, "0x%llx\n", atomic64_read(&allocator->ready_mem_size)); +} + +static ssize_t available_uncleared_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct conti_mem_allocator *allocator; + + allocator = container_of(kobj, struct conti_mem_allocator, kobj); + return sysfs_emit(buf, "0x%llx\n", atomic64_read(&allocator->uncleared_mem_size)); +} + +static struct kobj_attribute total_attr = __ATTR(total, 0400, total_show, NULL); +static struct kobj_attribute used_attr = __ATTR(used, 0400, used_show, NULL); +static struct kobj_attribute available_cleared_attr = + __ATTR(available_cleared, 0400, available_cleared_show, NULL); +static struct kobj_attribute available_uncleared_attr = + __ATTR(available_uncleared, 0400, available_uncleared_show, NULL); + +static struct attribute *conti_attrs[] = { + &total_attr.attr, + &used_attr.attr, + &available_cleared_attr.attr, + &available_uncleared_attr.attr, + NULL, +}; + +static struct attribute_group conti_attr_group = { + .attrs = conti_attrs, +}; + int conti_mem_allocator_init(struct conti_mem_allocator *allocator, int nid, size_t granu, - const struct conti_mempool_ops *ops, const char *fmt, ...) + const struct conti_mempool_ops *ops, + struct kobject *parent, const char *fmt, ...) { va_list ap; int ret; @@ -496,6 +604,8 @@ int conti_mem_allocator_init(struct conti_mem_allocator *allocator, int nid, siz return -EINVAL; } + memset(allocator, 0, sizeof(*allocator)); + va_start(ap, fmt); allocator->name = kvasprintf(GFP_KERNEL, fmt, ap); va_end(ap); @@ -506,36 +616,69 @@ int conti_mem_allocator_init(struct conti_mem_allocator *allocator, int nid, siz allocator->granu = granu; atomic64_set(&allocator->pooled_mem_size, 0); atomic64_set(&allocator->used_mem_size, 0); + atomic64_set(&allocator->ready_mem_size, 0); + atomic64_set(&allocator->uncleared_mem_size, 0); spin_lock_init(&allocator->lock); INIT_LIST_HEAD(&allocator->memseg_ready); init_waitqueue_head(&allocator->clear_wq); INIT_LIST_HEAD(&allocator->memseg_uncleared); - allocator->memseg_clearing = NULL; INIT_LIST_HEAD(&allocator->memseg_poisoned); - + allocator->memseg_clearing = NULL; allocator->ops = ops; if (ops->clear_memseg) { ret = clear_thread_init(allocator); - if (ret) { - kfree(allocator->name); - allocator->name = NULL; - return ret; - } + if (ret) + goto err_free_name; } ret = pool_thread_init(allocator); - if (ret) { - if (allocator->clear_work) - kthread_stop(allocator->clear_work); - kfree(allocator->name); - allocator->name = NULL; - return ret; + if (ret) + goto err_stop_clear_thread; + + /* Initialize sysfs */ + if (parent) { + ret = kobject_init_and_add(&allocator->kobj, &conti_ktype, + parent, "%s", allocator->name); + if (ret) { + pr_err("Failed to create sysfs directory for %s: %d\n", + allocator->name, ret); + goto err_stop_pool_thread; + } + + ret = sysfs_create_group(&allocator->kobj, &conti_attr_group); + if (ret) { + pr_err("Failed to create sysfs attributes for %s: %d\n", + allocator->name, ret); + goto err_cleanup_sysfs; + } + + allocator->sysfs_initialized = true; } allocator->initialized = true; - return 0; + +/* Exception paths: cleanup in reverse order of initialization */ +err_cleanup_sysfs: + kobject_put(&allocator->kobj); + +err_stop_pool_thread: + if (allocator->pool_work) { + kthread_stop(allocator->pool_work); + allocator->pool_work = NULL; + } + +err_stop_clear_thread: + if (allocator->clear_work) { + kthread_stop(allocator->clear_work); + allocator->clear_work = NULL; + } + +err_free_name: + kfree(allocator->name); + allocator->name = NULL; + return ret; } void conti_mem_allocator_deinit(struct conti_mem_allocator *allocator) @@ -544,17 +687,26 @@ void conti_mem_allocator_deinit(struct conti_mem_allocator *allocator) struct list_head free_list; unsigned long flags; + allocator->initialized = false; + INIT_LIST_HEAD(&free_list); - if (allocator->pool_work) + + if (allocator->pool_work) { kthread_stop(allocator->pool_work); + allocator->pool_work = NULL; + } - if (allocator->clear_work) + if (allocator->clear_work) { kthread_stop(allocator->clear_work); + allocator->clear_work = NULL; + } kfree(allocator->name); - if (!allocator->ops->pool_free_memseg) { + allocator->name = NULL; + + if (!allocator->ops || !allocator->ops->pool_free_memseg) { pr_err("pool_free_memseg is not defined.\n"); - return; + goto cleanup_sysfs; } /* Release all memory nodes chained in memseg_uncleared, memseg_ready @@ -572,5 +724,12 @@ void conti_mem_allocator_deinit(struct conti_mem_allocator *allocator) list_del(&node->list); conti_pool_free_memseg(allocator, node); } - memset(allocator, 0, sizeof(*allocator)); + +cleanup_sysfs: + if (allocator->sysfs_initialized) { + kobject_put(&allocator->kobj); + allocator->sysfs_initialized = false; + } + + allocator->memseg_clearing = NULL; } diff --git a/drivers/ub/obmm/conti_mem_allocator.h b/drivers/ub/obmm/conti_mem_allocator.h index 84cf24b64b4b62b3585f964ef39e881e018b7974..8e1255a091f06a2a204d07b5c4c0896571d3ec9d 100644 --- a/drivers/ub/obmm/conti_mem_allocator.h +++ b/drivers/ub/obmm/conti_mem_allocator.h @@ -11,6 +11,7 @@ #include #include #include +#include struct memseg_node { phys_addr_t addr; @@ -69,12 +70,16 @@ struct conti_mempool_ops { struct conti_mem_allocator { bool initialized; + bool sysfs_initialized; int nid; size_t granu; atomic64_t pooled_mem_size; atomic64_t used_mem_size; + atomic64_t ready_mem_size; /* cleared but not used (memseg_ready) */ + atomic64_t uncleared_mem_size; /* allocated but not cleared */ + /* (memseg_uncleared + memseg_clearing) */ spinlock_t lock; struct list_head memseg_ready; @@ -90,6 +95,9 @@ struct conti_mem_allocator { const struct conti_mempool_ops *ops; const char *name; + + /* sysfs support */ + struct kobject kobj; }; static inline size_t conti_get_total(struct conti_mem_allocator *a) @@ -103,7 +111,8 @@ static inline size_t conti_get_avail(struct conti_mem_allocator *a) } int conti_mem_allocator_init(struct conti_mem_allocator *allocator, int nid, size_t granu, - const struct conti_mempool_ops *ops, const char *fmt, ...); + const struct conti_mempool_ops *ops, + struct kobject *parent, const char *fmt, ...); void conti_mem_allocator_deinit(struct conti_mem_allocator *allocator); void conti_free_memory(struct conti_mem_allocator *allocator, struct list_head *head); diff --git a/drivers/ub/obmm/obmm_addr_check.c b/drivers/ub/obmm/obmm_addr_check.c index 4619dd9e6c6b01785aae269c813624615d153992..9122f0a8f3fae8a54bcfe50db4c6b60cae59b9f6 100644 --- a/drivers/ub/obmm/obmm_addr_check.c +++ b/drivers/ub/obmm/obmm_addr_check.c @@ -50,7 +50,6 @@ int occupy_pa_range(const struct obmm_pa_range *pa_range) int free_pa_range(const struct obmm_pa_range *pa_range) { int ret; - const char *user; void *entry; unsigned long flags; @@ -67,8 +66,6 @@ int free_pa_range(const struct obmm_pa_range *pa_range) pr_err("BUG: PA range to be freed does not fully match.\n"); ret = -ENOTRECOVERABLE; } - user = ((struct obmm_pa_range *)entry)->info.user == OBMM_ADDR_USER_DIRECT_IMPORT ? - "import" : "preimport"; kfree(entry); return ret; } diff --git a/drivers/ub/obmm/obmm_cache.c b/drivers/ub/obmm/obmm_cache.c index 1909da83de4c6b1e657df322484c52f17c4a1624..eab31a8df6726f44f1dfe9a31cae01fa5a57b84b 100644 --- a/drivers/ub/obmm/obmm_cache.c +++ b/drivers/ub/obmm/obmm_cache.c @@ -22,12 +22,6 @@ module_param(skip_cache_maintain, bool, 0444); MODULE_PARM_DESC(skip_cache_maintain, "Whether to skip cache maintain operation (to suppress errors in simulations)."); -static bool is_valid_cache_ops(unsigned long cache_ops) -{ - return cache_ops == OBMM_SHM_CACHE_NONE || cache_ops == OBMM_SHM_CACHE_INVAL || - cache_ops == OBMM_SHM_CACHE_WB_ONLY || cache_ops == OBMM_SHM_CACHE_WB_INVAL; -} - #define UB_MEM_DRAIN_TMOUT_MSEC 1000 int ub_write_queue_flush(uint32_t scna) @@ -75,10 +69,8 @@ int flush_cache_by_pa(phys_addr_t addr, size_t size, unsigned long cache_ops) return 0; } - if (!is_valid_cache_ops(cache_ops)) { - pr_err("invalid cache_ops %lu.\n", cache_ops); + if (!obmm_is_valid_cache_ops(cache_ops)) return -EINVAL; - } down(&sem); while (remain_size != 0) { @@ -126,10 +118,8 @@ int obmm_region_flush_range(struct obmm_region *reg, unsigned long offset, unsig struct obmm_export_region *e_reg; /* validation */ - if (!is_valid_cache_ops(cache_ops)) { - pr_err("invalid cache operation %u\n", cache_ops); + if (!obmm_is_valid_cache_ops(cache_ops)) return -EINVAL; - } if (offset >= reg->mem_size || length > reg->mem_size - offset || !IS_ALIGNED(offset, PAGE_SIZE) || !IS_ALIGNED(length, PAGE_SIZE)) { pr_err("invalid flush range for region=%d: offset=0x%lx, flush_length=0x%lx, region_length=0x%llx\n", diff --git a/drivers/ub/obmm/obmm_cache.h b/drivers/ub/obmm/obmm_cache.h index 28da446e0666cb2488f98057bfcad71c5bee55a6..a906d5d8f5e40de5fde5759d2343d5dec0f8e088 100644 --- a/drivers/ub/obmm/obmm_cache.h +++ b/drivers/ub/obmm/obmm_cache.h @@ -17,6 +17,15 @@ int flush_cache_by_pa(phys_addr_t addr, size_t size, unsigned long cache_ops); int obmm_region_flush_range(struct obmm_region *reg, unsigned long offset, unsigned long length, uint8_t cache_ops); void obmm_flush_tlb(struct mm_struct *mm); + +static inline bool obmm_is_valid_cache_ops(unsigned long cache_ops) +{ + if (cache_ops == OBMM_SHM_CACHE_NONE || cache_ops == OBMM_SHM_CACHE_INVAL || + cache_ops == OBMM_SHM_CACHE_WB_ONLY || cache_ops == OBMM_SHM_CACHE_WB_INVAL) + return true; + pr_err("Invalid cache operations: %#lx\n", cache_ops); + return false; +} /* Caller must guarantee that there is no concurrent modify requests made to the same va range. */ int modify_pgtable_prot(struct mm_struct *mm, void *va, size_t size, bool cacheable); int obmm_cache_clear(void); diff --git a/drivers/ub/obmm/obmm_core.h b/drivers/ub/obmm/obmm_core.h index ea6a1f6b5a82c93d4b9abd1a550056d12fe0f6b4..ec74887f8fe8fda16612d4c61043968f6befee5c 100644 --- a/drivers/ub/obmm/obmm_core.h +++ b/drivers/ub/obmm/obmm_core.h @@ -9,6 +9,7 @@ #define OBMM_CORE_H #include +#include #include #include #include @@ -112,8 +113,8 @@ struct obmm_region { * OBMM_MEM_ALLOW_NONCACHEABLE_MMAP: Supports non-cacheable mapping */ unsigned long mem_cap; - /* number of mmap */ - unsigned long mmap_count; + /* number of mmap (atomic to avoid deadlock in vma_open during split) */ + atomic_t mmap_count; struct obmm_ownership_info *ownership_info; /* protect ownership_info and serialize concurrent page table change requests */ diff --git a/drivers/ub/obmm/obmm_export.c b/drivers/ub/obmm/obmm_export.c index 237eb1e122f92a173fff41c1f9581a2431f8088f..f05afbcd58d49bc4a4e48c52f320fca31023411a 100644 --- a/drivers/ub/obmm/obmm_export.c +++ b/drivers/ub/obmm/obmm_export.c @@ -31,8 +31,10 @@ int export_flags_to_region_flags(unsigned long *region_flags, unsigned long user { *region_flags = 0; - if (user_flags & (~OBMM_EXPORT_FLAG_MASK)) + if (user_flags & (~OBMM_EXPORT_FLAG_MASK)) { + pr_err("undefined flags specified in export flags parameter %#lx.\n", user_flags); return -EINVAL; + } if (user_flags & OBMM_EXPORT_FLAG_ALLOW_MMAP) *region_flags |= OBMM_REGION_FLAG_ALLOW_MMAP; if (user_flags & OBMM_EXPORT_FLAG_FAST) @@ -69,7 +71,7 @@ static int setup_ummu(struct obmm_export_region *e_reg) return retval; /* register the memory region through UMMU */ - pr_info("call ummu_core_alloc_tdev(), priv_len=%u, tid=%u\n", attr.priv_len, tokenid); + pr_debug("call ummu_core_alloc_tdev(), priv_len=%u, tid=%u\n", attr.priv_len, tokenid); e_reg->ummu_dev = ummu_core_alloc_tdev(&attr, &tokenid); if (e_reg->ummu_dev == NULL) { pr_err("Failed to create UMMU device\n"); @@ -81,7 +83,7 @@ static int setup_ummu(struct obmm_export_region *e_reg) dev_name(e_reg->ummu_dev)); /* DMA mapping */ - pr_info("call dma_map_sgtable(..., dir=DMA_BIDIRECTIONAL, attrs=0)\n"); + pr_debug("call dma_map_sgtable(..., dir=DMA_BIDIRECTIONAL, attrs=0)\n"); retval = dma_map_sgtable(e_reg->ummu_dev, &e_reg->sgt, DMA_BIDIRECTIONAL, 0); if (retval) { pr_err("Failed to map sgtable on UMMU. ret=%pe\n", ERR_PTR(retval)); @@ -200,7 +202,7 @@ int obmm_unexport(const struct obmm_cmd_unexport *cmd_unexport) struct obmm_region *reg; struct obmm_export_region *e_reg; - pr_info("%s: mem_id=%llu, flags=%#llx.\n", __func__, cmd_unexport->mem_id, + pr_debug("%s: mem_id=%llu, flags=%#llx.\n", __func__, cmd_unexport->mem_id, cmd_unexport->flags); if (!validate_obmm_mem_id(cmd_unexport->mem_id)) return -ENOENT; @@ -228,7 +230,7 @@ int obmm_unexport(const struct obmm_cmd_unexport *cmd_unexport) uninit_obmm_region(reg); free_export_region(e_reg); - pr_info("%s: mem_id=%llu completed.\n", __func__, cmd_unexport->mem_id); + pr_debug("%s: mem_id=%llu completed.\n", __func__, cmd_unexport->mem_id); return 0; err_unexport_common: diff --git a/drivers/ub/obmm/obmm_export_from_pool.c b/drivers/ub/obmm/obmm_export_from_pool.c index daa4214955da9e615638a48f97829fba4d3dbf8c..2550a456c0d0eb6c4ca219bf710e204d32305e74 100644 --- a/drivers/ub/obmm/obmm_export_from_pool.c +++ b/drivers/ub/obmm/obmm_export_from_pool.c @@ -194,15 +194,16 @@ int alloc_export_memory_pool(struct obmm_export_region *e_reg) static int calculate_export_region_size(unsigned long *total_size, struct obmm_cmd_export *cmd_export) { - uint64_t i; nodemask_t nodes = NODE_MASK_NONE; + int i, min_numa_id = NUMA_NO_NODE, max_numa_id = 0; if (cmd_export->length > OBMM_MAX_LOCAL_NUMA_NODES) { pr_err("Size list is too long: max=%d, actual_length=%lld\n", OBMM_MAX_LOCAL_NUMA_NODES, cmd_export->length); return -E2BIG; } - if (cmd_export->pxm_numa > OBMM_MAX_LOCAL_NUMA_NODES) { + if (cmd_export->pxm_numa > OBMM_MAX_LOCAL_NUMA_NODES || + cmd_export->pxm_numa < 0) { pr_err("Invalid pxm_numa %d\n", cmd_export->pxm_numa); return -EINVAL; } @@ -224,6 +225,10 @@ static int calculate_export_region_size(unsigned long *total_size, pr_err("Memory size overflowed!\n"); return -EOVERFLOW; } + if (min_numa_id == NUMA_NO_NODE) + min_numa_id = i; + + max_numa_id = i; *total_size += cmd_export->size[i]; node_set(i, nodes); } @@ -235,6 +240,8 @@ static int calculate_export_region_size(unsigned long *total_size, node_set(cmd_export->pxm_numa, nodes); if (!nodes_on_same_package(&nodes)) { pr_err("Cannot use memory from multiple sockets or memory and ub controller is from different sockets.\n"); + pr_err("memory numa id ranges: [%d, %d], ub controller pxm: %d.\n", min_numa_id, + max_numa_id, cmd_export->pxm_numa); return -EINVAL; } @@ -287,18 +294,9 @@ static struct obmm_export_region *alloc_region_from_cmd(struct obmm_cmd_export * static void print_export_param(const struct obmm_cmd_export *cmd_export) { - unsigned int i; - - pr_info("obmm_export: len(sizes)=%#llx sizes={", cmd_export->length); - for (i = 0; i < cmd_export->length && i < OBMM_MAX_LOCAL_NUMA_NODES; i++) - if (cmd_export->size[i]) - pr_cont(" [%u]:%#llx", i, cmd_export->size[i]); - if (i < cmd_export->length) - pr_cont(" ..."); - - pr_cont(" } flags=%#llx deid=" EID_FMT64 " priv_len=%u\n", cmd_export->flags, - EID_ARGS64_H(cmd_export->deid), EID_ARGS64_L(cmd_export->deid), - cmd_export->priv_len); + pr_debug("obmm_export: len(sizes)=%#llx flags=%#llx deid=" EID_FMT64 " priv_len=%u\n", + cmd_export->length, cmd_export->flags, EID_ARGS64_H(cmd_export->deid), + EID_ARGS64_L(cmd_export->deid), cmd_export->priv_len); } /* obmm_export_from_pool: create an OBMM-exported memory region. The region is @@ -338,7 +336,7 @@ int obmm_export_from_pool(struct obmm_cmd_export *cmd_export) cmd_export->uba = uba; cmd_export->mem_id = mem_id; - pr_info("obmm_export: mem_id=%llu online.\n", mem_id); + pr_debug("obmm_export: mem_id=%llu online.\n", mem_id); return 0; out_unexport: diff --git a/drivers/ub/obmm/obmm_export_from_user.c b/drivers/ub/obmm/obmm_export_from_user.c index f136d3398753a3eb97e36a003c18f592b65e53d0..43fe7e41b948fae5e0a24ba6c332eb610e54909a 100644 --- a/drivers/ub/obmm/obmm_export_from_user.c +++ b/drivers/ub/obmm/obmm_export_from_user.c @@ -112,7 +112,8 @@ static bool hisi_workarounds_check_page_list(struct obmm_export_region *reg, str pr_debug("Page resides in node %u\n", node); reg->node_count = node + 1; } - if (reg->affinity > OBMM_MAX_LOCAL_NUMA_NODES) { + if (reg->affinity > OBMM_MAX_LOCAL_NUMA_NODES || + reg->affinity < 0) { pr_err("Invalid pxm_numa %d\n", reg->affinity); return false; } @@ -313,7 +314,7 @@ alloc_export_region_from_obmm_cmd_export_pid(const struct obmm_cmd_export_pid *e static void print_export_pid_param(const struct obmm_cmd_export_pid *cmd_export_pid) { - pr_info("obmm_export_useraddr: pid=%d length=%#llx priv_len=%u deid=" + pr_debug("obmm_export_useraddr: pid=%d length=%#llx priv_len=%u deid=" EID_FMT64 " vendor_len=%u\n", cmd_export_pid->pid, cmd_export_pid->length, cmd_export_pid->priv_len, EID_ARGS64_H(cmd_export_pid->deid), EID_ARGS64_L(cmd_export_pid->deid), @@ -357,7 +358,7 @@ int obmm_export_pid(struct obmm_cmd_export_pid *export_pid) export_pid->uba = uba; export_pid->mem_id = mem_id; - pr_info("obmm_export_useraddr: mem_id=%llu online.\n", mem_id); + pr_debug("obmm_export_useraddr: mem_id=%llu online.\n", mem_id); return 0; out_unexport: diff --git a/drivers/ub/obmm/obmm_export_region_ops.c b/drivers/ub/obmm/obmm_export_region_ops.c index b561224b660bccafd42618c97690168f96789dba..89b7dadd3d767eb8ba9f26e4bbb57d4cfb1e5ac5 100644 --- a/drivers/ub/obmm/obmm_export_region_ops.c +++ b/drivers/ub/obmm/obmm_export_region_ops.c @@ -163,7 +163,6 @@ int kernel_pgtable_set_export_invalid(struct obmm_export_region *e_reg, unsigned struct map_range_info { struct obmm_export_region *e_reg; struct vm_area_struct *vma; - unsigned long orig_pgoff; enum obmm_mmap_granu mmap_granu; int ret; }; @@ -174,14 +173,7 @@ static int map_range_call(phys_addr_t start, phys_addr_t end, unsigned long offs unsigned long length = end - start + 1; unsigned long vm_offset; - /* - * The offset is offset_from_head. - * - * In the case of private mapping, after remap_pfn_range is called, - * vma->vm_pgoff will be set to pfn, - * but we still need the original offset relative to the start of the region. - */ - vm_offset = offset - (info->orig_pgoff << PAGE_SHIFT); + vm_offset = offset - (info->vma->vm_pgoff << PAGE_SHIFT); if (info->mmap_granu == OBMM_MMAP_GRANU_PAGE) { info->ret = remap_pfn_range(info->vma, info->vma->vm_start + vm_offset, start >> PAGE_SHIFT, length, info->vma->vm_page_prot); @@ -211,7 +203,6 @@ int map_export_region(struct vm_area_struct *vma, struct obmm_export_region *e_r info.e_reg = e_reg; info.vma = vma; info.ret = 0; - info.orig_pgoff = vma->vm_pgoff; info.mmap_granu = mmap_granu; size = vma->vm_end - vma->vm_start; diff --git a/drivers/ub/obmm/obmm_import.c b/drivers/ub/obmm/obmm_import.c index 8982f1f3dfab985448f544e1bb3cd454f588a3c5..5721065031d54035c46212d400b4e6586e8a81a3 100644 --- a/drivers/ub/obmm/obmm_import.c +++ b/drivers/ub/obmm/obmm_import.c @@ -102,7 +102,7 @@ static int teardown_remote_numa(struct obmm_import_region *i_reg, bool force) if (ret) return ret; - pr_info("call external: remove_memory_remote(nid=%d, size=%#llx)\n", + pr_debug("call external: remove_memory_remote(nid=%d, size=%#llx)\n", i_reg->numa_id, i_reg->region.mem_size); ret = remove_memory_remote(i_reg->numa_id, i_reg->pa, i_reg->region.mem_size); pr_debug("external called: remove_memory_remote, ret=%pe\n", ERR_PTR(ret)); @@ -114,7 +114,7 @@ static int teardown_remote_numa(struct obmm_import_region *i_reg, bool force) } if (region_preimport(&i_reg->region)) { - pr_info("call external: add_memory_remote(nid=%d, size=0x%llx, flags=MEMORY_KEEP_ISOLATED)\n", + pr_debug("call external: add_memory_remote(nid=%d, size=0x%llx, flags=MEMORY_KEEP_ISOLATED)\n", i_reg->numa_id, i_reg->region.mem_size); this_ret = add_memory_remote(i_reg->numa_id, i_reg->pa, i_reg->region.mem_size, MEMORY_KEEP_ISOLATED); @@ -144,7 +144,7 @@ static int setup_remote_numa(struct obmm_import_region *i_reg) return -EINVAL; } - pr_info("call external: add_memory_remote(nid=%d, flags=%d)\n", + pr_debug("call external: add_memory_remote(nid=%d, flags=%d)\n", i_reg->numa_id, flags); ret = add_memory_remote(i_reg->numa_id, i_reg->pa, i_reg->region.mem_size, flags); pr_debug("external called: add_memory_remote() returned %d\n", ret); @@ -454,7 +454,7 @@ static int init_import_region_from_cmd(const struct obmm_cmd_import *param, static void print_import_param(const struct obmm_cmd_import *cmd_import) { - pr_info("obmm_import: scna=%#x {pa=%#llx length=%#llx} flags=%#llx nid=%d base_dist=%u seid=" + pr_debug("obmm_import: scna=%#x {pa=%#llx length=%#llx} flags=%#llx nid=%d base_dist=%u seid=" EID_FMT64 " priv_len=%u\n", cmd_import->scna, cmd_import->addr, cmd_import->length, cmd_import->flags, cmd_import->numa_id, cmd_import->base_dist, EID_ARGS64_H(cmd_import->seid), @@ -504,7 +504,7 @@ int obmm_import(struct obmm_cmd_import *cmd_import) cmd_import->numa_id = numa_id; cmd_import->mem_id = mem_id; - pr_info("%s: mem_id=%llu online\n", __func__, cmd_import->mem_id); + pr_debug("%s: mem_id=%llu online\n", __func__, cmd_import->mem_id); return 0; out_release_memory: @@ -529,7 +529,7 @@ int obmm_unimport(const struct obmm_cmd_unimport *cmd_unimport) struct obmm_region *reg; struct obmm_import_region *i_reg; - pr_info("%s: mem_id=%llu, flags=%#llx.\n", __func__, cmd_unimport->mem_id, + pr_debug("%s: mem_id=%llu, flags=%#llx.\n", __func__, cmd_unimport->mem_id, cmd_unimport->flags); if (!validate_obmm_mem_id(cmd_unimport->mem_id)) return -ENOENT; @@ -557,7 +557,7 @@ int obmm_unimport(const struct obmm_cmd_unimport *cmd_unimport) wait_until_dev_released(&i_reg->region); kfree(i_reg); - pr_info("%s: mem_id=%llu completed.\n", __func__, cmd_unimport->mem_id); + pr_debug("%s: mem_id=%llu completed.\n", __func__, cmd_unimport->mem_id); return 0; err_unimport: diff --git a/drivers/ub/obmm/obmm_ownership.c b/drivers/ub/obmm/obmm_ownership.c index 625446b240646c8a7091af3818826083f3d35148..8b7071895637da8e7a7d8fd838ae433fadf7d63e 100644 --- a/drivers/ub/obmm/obmm_ownership.c +++ b/drivers/ub/obmm/obmm_ownership.c @@ -1,366 +1,143 @@ // SPDX-License-Identifier: GPL-2.0+ /* * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * Description：OBMM Framework's implementations. + * Description: OBMM ownership tracking - region level R/W counters */ -#include +#include #include +#include "obmm_cache.h" #include "obmm_core.h" #include "obmm_ownership.h" -static inline uint32_t merge_counts(uint32_t read, uint32_t write) +/* Merge cache ops: WB_ONLY + INVAL = WB_INVAL, others return stronger op */ +uint8_t merge_cache_ops(uint8_t ops1, uint8_t ops2) { - return (read << READ_SHIFT) | (write << WRITE_SHIFT); + if (ops1 == OBMM_SHM_CACHE_NONE) + return ops2; + if (ops2 == OBMM_SHM_CACHE_NONE) + return ops1; + if (ops1 == OBMM_SHM_CACHE_WB_INVAL || ops2 == OBMM_SHM_CACHE_WB_INVAL) + return OBMM_SHM_CACHE_WB_INVAL; + if ((ops1 == OBMM_SHM_CACHE_WB_ONLY && ops2 == OBMM_SHM_CACHE_INVAL) || + (ops1 == OBMM_SHM_CACHE_INVAL && ops2 == OBMM_SHM_CACHE_WB_ONLY)) + return OBMM_SHM_CACHE_WB_INVAL; + return ops1; } -/* - * dirty -> non-dirty: INVAL_WB - * non-dirty cacheable -> NC: INVAL - * cache capability rise: NONE - * cache operation coverage: INVAL_WB > INVAL > NONE - */ -uint8_t infer_cache_ops(uint8_t cur_state, uint8_t target_state) +/* Update R/W counters for a page, return cache op needed */ +static uint8_t update_page_ownership(struct obmm_ownership_info *ownership, + unsigned long pgoff, + uint8_t old_access, + uint8_t new_access) { - bool cur_dirty, cur_none, target_dirty, target_none, target_clean; - uint8_t ops = OBMM_SHM_CACHE_NONE; - - cur_dirty = ((cur_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READWRITE && - (cur_state & OBMM_SHM_MEM_CACHE_MASK) == OBMM_SHM_MEM_NORMAL); - target_dirty = ((target_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READWRITE && - (target_state & OBMM_SHM_MEM_CACHE_MASK) == OBMM_SHM_MEM_NORMAL); - target_clean = ((target_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READONLY && - (target_state & OBMM_SHM_MEM_CACHE_MASK) == OBMM_SHM_MEM_NORMAL); - cur_none = ((cur_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_NO_ACCESS || - (cur_state & OBMM_SHM_MEM_CACHE_MASK) != OBMM_SHM_MEM_NORMAL); - target_none = ((target_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_NO_ACCESS || - (target_state & OBMM_SHM_MEM_CACHE_MASK) != OBMM_SHM_MEM_NORMAL); - if (cur_dirty && target_clean) - ops = OBMM_SHM_CACHE_WB_ONLY; - else if (cur_dirty && !target_dirty) - ops = OBMM_SHM_CACHE_WB_INVAL; - else if (!cur_none && target_none) - ops = OBMM_SHM_CACHE_INVAL; - - pr_debug("%s: target_state = %u; ops = %u\n", __func__, target_state, ops); - return ops; + struct obmm_page_state *page = &ownership->page_states[pgoff]; + uint16_t old_w = page->w_count; + uint16_t old_r = page->r_count; + uint16_t new_w, new_r; + + if (obmm_access_is_writer(old_access)) + page->w_count--; + if (obmm_access_is_reader(old_access)) + page->r_count--; + if (obmm_access_is_writer(new_access)) + page->w_count++; + if (obmm_access_is_reader(new_access)) + page->r_count++; + + new_w = page->w_count; + new_r = page->r_count; + + if (old_w > 0 && new_w == 0) + return new_r > 0 ? OBMM_SHM_CACHE_WB_ONLY : OBMM_SHM_CACHE_WB_INVAL; + if (old_r > 0 && new_r == 0 && new_w == 0) + return OBMM_SHM_CACHE_INVAL; + return OBMM_SHM_CACHE_NONE; } -/** - * Calculate the local page state index corresponding to the VMA address - */ -int vma_addr_to_page_idx_local(struct vm_area_struct *vma, unsigned long addr) +/* Update counters for VMA range, return combined cache op */ +uint8_t update_vma_perm_count(struct obmm_region *reg, + unsigned long region_pgoff, + unsigned long npages, + uint8_t old_access, + uint8_t new_access) { - unsigned long offset_in_vma = addr - vma->vm_start; + struct obmm_ownership_info *ownership = reg->ownership_info; + uint8_t combined_ops = OBMM_SHM_CACHE_NONE, op; + unsigned long start_idx, nentries, i; - return offset_in_vma >> PAGE_SHIFT; -} + if (!ownership) + return OBMM_SHM_CACHE_NONE; -/** - * Calculate the global page state index corresponding to the VMA address - */ -static int vma_addr_to_page_idx(struct vm_area_struct *vma, - struct obmm_local_state_info *local_state_info, unsigned long addr) -{ - return local_state_info->orig_pgoff + vma_addr_to_page_idx_local(vma, addr); -} - -/* Check if new permissions conflict with existing mappings */ -static int check_target_state_allowed(uint32_t state_count, uint8_t target_mem_state) -{ - uint32_t read_count, write_count; - - read_count = GET_R_COUNTER(state_count); - write_count = GET_W_COUNTER(state_count); + start_idx = ownership_pgoff_to_index(reg, region_pgoff); + nentries = ownership_size_to_nentries(reg, npages << PAGE_SHIFT); - switch (target_mem_state & OBMM_SHM_MEM_ACCESS_MASK) { - case OBMM_SHM_MEM_READONLY: - fallthrough; - case OBMM_SHM_MEM_READEXEC: - if (read_count == MAX_READ_COUNT) { - pr_warn("%s: readonly map failed, read_count=%d\n", __func__, read_count); - return -EBUSY; - } - break; - case OBMM_SHM_MEM_READWRITE: - if (write_count == MAX_WRITE_COUNT) { - pr_warn("%s: readwrite map failed, write_count=%d\n", __func__, - write_count); - return -EBUSY; - } - break; - default: - break; + for (i = 0; i < nentries; i++) { + op = update_page_ownership(ownership, start_idx + i, old_access, new_access); + combined_ops = merge_cache_ops(combined_ops, op); } - return 0; -} - -/** - * Check whether mmap operation is possible. - * The caller holds region state_mutex lock. - */ -int check_mmap_allowed(struct obmm_region *reg, struct vm_area_struct *vma, uint8_t mem_state) -{ - int idx_offset, page_idx_start, page_count, ret; - uint32_t state_count; - struct obmm_local_state_info *local_state_info; - struct obmm_ownership_info *info; - info = reg->ownership_info; - local_state_info = (struct obmm_local_state_info *)vma->vm_private_data; - page_idx_start = vma_addr_to_page_idx(vma, local_state_info, vma->vm_start); - page_count = local_state_info->npages; - - for (idx_offset = 0; idx_offset < page_count; idx_offset++) { - state_count = info->mem_state_arr[page_idx_start + idx_offset]; - ret = check_target_state_allowed(state_count, mem_state); - if (ret) - return ret; - } - return 0; + return combined_ops; } -/* - * Update the count of the corresponding permission in the state. - */ -static uint32_t update_state_count(uint32_t state_count, uint8_t target_mem_state, bool inc) +/* Convert access bits to vm_flags (R/W only, preserves existing EXEC) */ +vm_flags_t access_to_vm_flags(uint8_t access) { - uint32_t read_count, write_count; - int delta; - - delta = inc ? 1 : -1; - read_count = GET_R_COUNTER(state_count); - write_count = GET_W_COUNTER(state_count); - - /* inc new permission count */ - switch (target_mem_state & OBMM_SHM_MEM_ACCESS_MASK) { - case OBMM_SHM_MEM_NO_ACCESS: - break; - case OBMM_SHM_MEM_READONLY: - fallthrough; - case OBMM_SHM_MEM_READEXEC: - read_count += delta; - break; + switch (access & OBMM_SHM_MEM_ACCESS_MASK) { case OBMM_SHM_MEM_READWRITE: - write_count += delta; - break; + return VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + case OBMM_SHM_MEM_READEXEC: + return VM_READ | VM_MAYREAD; + case OBMM_SHM_MEM_READONLY: + return VM_READ | VM_MAYREAD; default: - break; - } - return merge_counts(read_count, write_count); -} - -/** - * Check whether permissions can be modified. - * The caller holds region state_mutex lock. - */ -int check_modify_ownership_allowed(struct obmm_region *reg, struct vm_area_struct *vma, - const struct obmm_cmd_update_range *update_info) -{ - int idx_offset, page_idx_start, page_count, local_page_idx_start, ret; - uint32_t state_count; - struct obmm_local_state_info *local_state_info; - struct obmm_ownership_info *info; - uint8_t old_state; - - info = reg->ownership_info; - local_state_info = (struct obmm_local_state_info *)vma->vm_private_data; - - page_idx_start = vma_addr_to_page_idx(vma, local_state_info, update_info->start); - local_page_idx_start = vma_addr_to_page_idx_local(vma, update_info->start); - page_count = (update_info->end - update_info->start) >> PAGE_SHIFT; - - for (idx_offset = 0; idx_offset < page_count; idx_offset++) { - old_state = - local_state_info->local_mem_state_arr[local_page_idx_start + idx_offset]; - state_count = info->mem_state_arr[page_idx_start + idx_offset]; - - /* Check for conflicts after simulating permission changes */ - /* Remove old permissions */ - state_count = update_state_count(state_count, old_state, false); - ret = check_target_state_allowed(state_count, update_info->mem_state); - if (ret) - return ret; - } - - return 0; -} - -/** - * Increase global page permission count (for mmap). - * The caller holds region state_mutex lock. - */ -void add_mapping_permission(struct obmm_region *reg, struct vm_area_struct *vma, uint8_t mem_state) -{ - int idx_offset, page_idx_start, page_count; - uint32_t state_count; - struct obmm_local_state_info *local_state_info; - struct obmm_ownership_info *info; - - info = reg->ownership_info; - local_state_info = (struct obmm_local_state_info *)vma->vm_private_data; - page_idx_start = vma_addr_to_page_idx(vma, local_state_info, vma->vm_start); - page_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; - - for (idx_offset = 0; idx_offset < page_count; idx_offset++) { - state_count = info->mem_state_arr[page_idx_start + idx_offset]; - state_count = update_state_count(state_count, mem_state, true); - info->mem_state_arr[page_idx_start + idx_offset] = state_count; - } -} - -/** - * Update global page permission count and VMA local permissions. - * The caller holds region state_mutex lock. - */ -void update_ownership(struct obmm_region *reg, struct vm_area_struct *vma, - const struct obmm_cmd_update_range *update_info) -{ - int idx_offset, page_idx_start, page_count, local_page_idx_start; - uint32_t state_count; - uint8_t old_state; - struct obmm_local_state_info *local_state_info; - struct obmm_ownership_info *info; - - info = reg->ownership_info; - local_state_info = (struct obmm_local_state_info *)vma->vm_private_data; - - page_idx_start = vma_addr_to_page_idx(vma, local_state_info, update_info->start); - local_page_idx_start = vma_addr_to_page_idx_local(vma, update_info->start); - page_count = (update_info->end - update_info->start) >> PAGE_SHIFT; - - for (idx_offset = 0; idx_offset < page_count; idx_offset++) { - old_state = - local_state_info->local_mem_state_arr[local_page_idx_start + idx_offset]; - - state_count = info->mem_state_arr[page_idx_start + idx_offset]; - /* Remove old permissions */ - state_count = update_state_count(state_count, old_state, false); - /* Add new permissions */ - state_count = update_state_count(state_count, update_info->mem_state, true); - - /* update mem_state_arr */ - info->mem_state_arr[page_idx_start + idx_offset] = state_count; - /* update vma local_state_info */ - local_state_info->local_mem_state_arr[local_page_idx_start + idx_offset] = - update_info->mem_state; - } -} - -/** - * Remove global page permission count. - * The caller holds region state_mutex lock. - */ -void remove_mapping_permission(struct obmm_region *reg, struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - int idx_offset, page_idx_start, page_count, local_page_idx_start; - uint32_t state_count; - uint8_t old_state; - struct obmm_local_state_info *local_state_info; - struct obmm_ownership_info *info; - - info = reg->ownership_info; - local_state_info = (struct obmm_local_state_info *)vma->vm_private_data; - - page_idx_start = vma_addr_to_page_idx(vma, local_state_info, start); - local_page_idx_start = vma_addr_to_page_idx_local(vma, start); - page_count = (end - start) >> PAGE_SHIFT; - - for (idx_offset = 0; idx_offset < page_count; idx_offset++) { - old_state = - local_state_info->local_mem_state_arr[local_page_idx_start + idx_offset]; - state_count = info->mem_state_arr[page_idx_start + idx_offset]; - - /* Remove permissions */ - state_count = update_state_count(state_count, old_state, false); - info->mem_state_arr[page_idx_start + idx_offset] = state_count; - } -} - -int init_local_state_info(struct vm_area_struct *vma, uint8_t mem_state) -{ - struct obmm_local_state_info *local_state_info; - unsigned long size; - int ret, i; - - size = vma->vm_end - vma->vm_start; - local_state_info = kzalloc(sizeof(struct obmm_local_state_info), GFP_KERNEL); - if (local_state_info == NULL) - return -ENOMEM; - - local_state_info->npages = size >> PAGE_SHIFT; - local_state_info->local_mem_state_arr = vmalloc(sizeof(uint8_t) * local_state_info->npages); - - if (local_state_info->local_mem_state_arr == NULL) { - ret = -ENOMEM; - goto out_local_state_info; + return 0; } - for (i = 0; i < local_state_info->npages; i++) - local_state_info->local_mem_state_arr[i] = mem_state; - - local_state_info->orig_pgoff = vma->vm_pgoff; - vma->vm_private_data = local_state_info; - - pr_debug("init vma local state: npages=%d, state=%#x\n", local_state_info->npages, - mem_state); - return 0; -out_local_state_info: - kfree(local_state_info); - return ret; -} - -void release_local_state_info(struct vm_area_struct *vma) -{ - struct obmm_local_state_info *local_state_info; - - local_state_info = (struct obmm_local_state_info *)vma->vm_private_data; - - vma->vm_private_data = NULL; - vfree(local_state_info->local_mem_state_arr); - kfree(local_state_info); } -/* - * Initialize the global page permission count array. - * The obmm_ownership_info is created when the region is mmapped for the first time, - * so the caller need to hold region state_mutex lock. - */ int init_ownership_info(struct obmm_region *reg) { struct obmm_ownership_info *info; - int i, ret; + unsigned long nentries; if (reg->ownership_info) return 0; - info = kzalloc(sizeof(struct obmm_ownership_info), GFP_KERNEL); - if (info == NULL) - return -ENOMEM; - info->npages = reg->mem_size >> PAGE_SHIFT; - info->mem_state_arr = vmalloc(sizeof(uint32_t) * info->npages); - if (info->mem_state_arr == NULL) { - ret = -ENOMEM; - goto out_free_info; + if (reg->mmap_granu == OBMM_MMAP_GRANU_NONE) { + pr_err("init ownership: mmap_granu not set\n"); + return -EINVAL; } - for (i = 0; i < info->npages; i++) - info->mem_state_arr[i] = 0; + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + nentries = ownership_size_to_nentries(reg, reg->mem_size); + info->page_states = vzalloc(sizeof(struct obmm_page_state) * nentries); + if (!info->page_states) + goto err_pages; + + /* mem_size >> PMD_SHIFT always fits in int */ + info->nentries = (int)nentries; reg->ownership_info = info; - pr_debug("init ownership: npages=%d, state=%#x\n", info->npages, 0U); + pr_debug("init ownership: granu=%d nentries=%lu\n", reg->mmap_granu, nentries); return 0; -out_free_info: + +err_pages: kfree(info); - return ret; + return -ENOMEM; } void release_ownership_info(struct obmm_region *reg) { struct obmm_ownership_info *info = reg->ownership_info; + if (!info) + return; + reg->ownership_info = NULL; - vfree(info->mem_state_arr); + vfree(info->page_states); kfree(info); } diff --git a/drivers/ub/obmm/obmm_ownership.h b/drivers/ub/obmm/obmm_ownership.h index a4c49abb81170db6f18645b7d6fada73a42f9911..48d928358c978911fcb86aaba35b8509a8a9ec17 100644 --- a/drivers/ub/obmm/obmm_ownership.h +++ b/drivers/ub/obmm/obmm_ownership.h @@ -8,49 +8,83 @@ #include "obmm_core.h" -#define WRITE_COUNT_BIT 16 -#define READ_COUNT_BIT 16 +/* Per-page ownership counters - r/w kept together for cache locality */ +struct obmm_page_state { + uint16_t r_count; + uint16_t w_count; +}; -#define WRITE_MASK ((1 << WRITE_COUNT_BIT) - 1) /* 16-bit mask */ -#define READ_MASK ((1 << READ_COUNT_BIT) - 1) /* 16-bit mask */ +struct obmm_ownership_info { + struct obmm_page_state *page_states; + int nentries; +}; -#define MAX_WRITE_COUNT WRITE_MASK -#define MAX_READ_COUNT READ_MASK +/* Extract access bits from vm_flags using uapi OBMM_SHM_MEM_* definitions */ +static inline uint8_t vm_flags_to_access(vm_flags_t flags) +{ + if (flags & VM_WRITE) + return OBMM_SHM_MEM_READWRITE; + if (flags & VM_READ) + return (flags & VM_EXEC) ? OBMM_SHM_MEM_READEXEC : OBMM_SHM_MEM_READONLY; + return OBMM_SHM_MEM_NO_ACCESS; +} -#define WRITE_SHIFT 0 -#define READ_SHIFT (WRITE_COUNT_BIT) +static inline bool obmm_access_is_reader(uint8_t access) +{ + return (access & OBMM_SHM_MEM_ACCESS_MASK) != OBMM_SHM_MEM_NO_ACCESS; +} -#define GET_W_COUNTER(val) (((val) >> WRITE_SHIFT) & WRITE_MASK) -#define GET_R_COUNTER(val) (((val) >> READ_SHIFT) & READ_MASK) +static inline bool obmm_access_is_writer(uint8_t access) +{ + return (access & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READWRITE; +} -/* - * [ 16-31 : 0-15 ] - * state:[ Read : Write ] - * [ 65535 : 65535 ] - */ -struct obmm_ownership_info { - uint32_t *mem_state_arr; - int npages; -}; +/* Convert pgoff to ownership array index based on mmap granularity */ +static inline unsigned long ownership_pgoff_to_index(const struct obmm_region *reg, + unsigned long pgoff) +{ + if (reg->mmap_granu == OBMM_MMAP_GRANU_PMD) + return pgoff >> (PMD_SHIFT - PAGE_SHIFT); + return pgoff; +} -struct obmm_local_state_info { - uint8_t *local_mem_state_arr; - /* Original file offset in vma */ - unsigned long orig_pgoff; - int npages; -}; -int vma_addr_to_page_idx_local(struct vm_area_struct *vma, unsigned long addr); -uint8_t infer_cache_ops(uint8_t cur_state, uint8_t target_state); +/* Convert size in bytes to ownership array entry count */ +static inline unsigned long ownership_size_to_nentries(const struct obmm_region *reg, + unsigned long size) +{ + if (reg->mmap_granu == OBMM_MMAP_GRANU_PMD) + return size >> PMD_SHIFT; + return size >> PAGE_SHIFT; +} + +/* Check if address is aligned according to mmap granularity */ +static inline bool obmm_is_aligned(const struct obmm_region *reg, unsigned long addr) +{ + if (reg->mmap_granu == OBMM_MMAP_GRANU_PMD) + return IS_ALIGNED(addr, PMD_SIZE); + return IS_ALIGNED(addr, PAGE_SIZE); +} + +/* Convert VM flags to mem_state */ +static inline unsigned long vm_flags_to_mem_state(vm_flags_t vm_flags, bool cacheable) +{ + unsigned long access = vm_flags_to_access(vm_flags); + + if (access != OBMM_SHM_MEM_NO_ACCESS) + access |= cacheable ? OBMM_SHM_MEM_NORMAL : OBMM_SHM_MEM_NORMAL_NC; + + return access; +} + +/* Convert access bits to vm_flags (R/W only, preserves existing EXEC) */ +vm_flags_t access_to_vm_flags(uint8_t access); +uint8_t merge_cache_ops(uint8_t ops1, uint8_t ops2); +uint8_t update_vma_perm_count(struct obmm_region *reg, + unsigned long region_pgoff, + unsigned long npages, + uint8_t old_access, + uint8_t new_access); int init_ownership_info(struct obmm_region *reg); -int init_local_state_info(struct vm_area_struct *vma, uint8_t mem_state); void release_ownership_info(struct obmm_region *reg); -void release_local_state_info(struct vm_area_struct *vma); -void add_mapping_permission(struct obmm_region *reg, struct vm_area_struct *vma, uint8_t mem_state); -void update_ownership(struct obmm_region *reg, struct vm_area_struct *vma, - const struct obmm_cmd_update_range *update_info); -int check_modify_ownership_allowed(struct obmm_region *reg, struct vm_area_struct *vma, - const struct obmm_cmd_update_range *update_info); -int check_mmap_allowed(struct obmm_region *reg, struct vm_area_struct *vma, uint8_t mem_state); -void remove_mapping_permission(struct obmm_region *reg, struct vm_area_struct *vma, - unsigned long start, unsigned long end); + #endif diff --git a/drivers/ub/obmm/obmm_preimport.c b/drivers/ub/obmm/obmm_preimport.c index aac211232a881c9235284d5fa14352fd64abdbbb..c723e9af66cf335b208ea016c027184ea67ffa36 100644 --- a/drivers/ub/obmm/obmm_preimport.c +++ b/drivers/ub/obmm/obmm_preimport.c @@ -107,7 +107,7 @@ int preimport_prepare_common(struct preimport_range *pr, uint8_t base_dist) return -EINVAL; } - pr_info("call external: add_memory_remote(nid=%d, flags=MEMORY_KEEP_ISOLATED)\n", + pr_debug("call external: add_memory_remote(nid=%d, flags=MEMORY_KEEP_ISOLATED)\n", pr->numa_id); ret = add_memory_remote(pr->numa_id, pr->start, pr->end - pr->start + 1, MEMORY_KEEP_ISOLATED); @@ -133,7 +133,7 @@ int preimport_prepare_common(struct preimport_range *pr, uint8_t base_dist) return 0; err_remove_memory_remote: - pr_info("call external: remove_memory_remote(nid=%d)\n", pr->numa_id); + pr_debug("call external: remove_memory_remote(nid=%d)\n", pr->numa_id); ret_err = remove_memory_remote(pr->numa_id, pr->start, pr->end - pr->start + 1); pr_debug("external called: remove_memory_remote() returned %d\n", ret_err); return ret; @@ -143,7 +143,7 @@ int preimport_release_common(struct preimport_range *pr, bool force) { int ret; - pr_info("call external: remove_memory_remote(nid=%d)\n", pr->numa_id); + pr_debug("call external: remove_memory_remote(nid=%d)\n", pr->numa_id); ret = remove_memory_remote(pr->numa_id, pr->start, pr->end - pr->start + 1); pr_debug("external called: remove_memory_remote() returned %pe\n", ERR_PTR(ret)); if (ret && !force) { @@ -184,7 +184,7 @@ int check_preimport_datapath_common(const struct preimport_range *pr, static void print_preimport_param(const struct obmm_cmd_preimport *cmd) { - pr_info("obmm_preimport: pa=%#llx length=%#llx scna=%#x dcna=%#x flags=%#llx nid=%d base_dist=%u deid=" + pr_debug("obmm_preimport: pa=%#llx length=%#llx scna=%#x dcna=%#x flags=%#llx nid=%d base_dist=%u deid=" EID_FMT64 " seid=" EID_FMT64 " priv_len=%u\n", cmd->pa, cmd->length, cmd->scna, cmd->dcna, cmd->flags, cmd->numa_id, cmd->base_dist, EID_ARGS64_H(cmd->deid), EID_ARGS64_L(cmd->deid), @@ -206,7 +206,7 @@ int obmm_preimport(struct obmm_cmd_preimport *cmd) if (ret) module_put(THIS_MODULE); else - pr_info("%s: preimport on nid=%d finished.\n", __func__, cmd->numa_id); + pr_debug("%s: preimport on nid=%d finished.\n", __func__, cmd->numa_id); return ret; } @@ -221,7 +221,7 @@ static int check_unpreimport_cmd_common(const struct obmm_cmd_preimport *cmd) static void print_unpreimport_param(const struct obmm_cmd_preimport *cmd) { - pr_info("obmm_unpreimport: pa=%#llx, length=%#llx.\n", cmd->pa, cmd->length); + pr_debug("obmm_unpreimport: pa=%#llx, length=%#llx.\n", cmd->pa, cmd->length); } int obmm_unpreimport(struct obmm_cmd_preimport *cmd) @@ -236,7 +236,7 @@ int obmm_unpreimport(struct obmm_cmd_preimport *cmd) ret = preimport_release_prefilled(cmd->pa, cmd->pa + cmd->length - 1); if (ret == 0) module_put(THIS_MODULE); - pr_info("%s: unpreimport on pa=%#llx finished.\n", __func__, cmd->pa); + pr_debug("%s: unpreimport on pa=%#llx finished.\n", __func__, cmd->pa); return ret; } diff --git a/drivers/ub/obmm/obmm_shm_dev.c b/drivers/ub/obmm/obmm_shm_dev.c index 0814c37d12b46ca600fb95f5a96868548003d44d..a46cc4e9df9c62814e45fb075b3c2c2d9f1597cc 100644 --- a/drivers/ub/obmm/obmm_shm_dev.c +++ b/drivers/ub/obmm/obmm_shm_dev.c @@ -4,10 +4,14 @@ * Description：OBMM Framework's implementations. */ -#include #include +#include #include #include +#include +#include +#include +#include #include "obmm_cache.h" #include "obmm_sysfs.h" @@ -16,88 +20,58 @@ #include "obmm_ownership.h" #include "obmm_shm_dev.h" +#include + static dev_t obmm_devt; static const char *obmm_shm_region_name = "OBMM_SHMDEV"; static const char *obmm_shm_rootdev_name = "obmm"; static struct device *obmm_shm_rootdev; -static int scan_and_flush(struct obmm_region *reg, struct vm_area_struct *vma, - const struct obmm_cmd_update_range *update_info); - -/** - * Convert VM flags to mem state - */ -static unsigned long get_vma_mem_state(const vm_flags_t vm_flags, bool cacheable) -{ - unsigned long mem_state; - - if (vm_flags & VM_WRITE) - mem_state = OBMM_SHM_MEM_READWRITE; - else if ((vm_flags & VM_READ) && (vm_flags & VM_EXEC)) - mem_state = OBMM_SHM_MEM_READEXEC; - else if (vm_flags & VM_READ) - mem_state = OBMM_SHM_MEM_READONLY; - else - mem_state = OBMM_SHM_MEM_NO_ACCESS; - - if (cacheable && mem_state != OBMM_SHM_MEM_NO_ACCESS) - mem_state |= OBMM_SHM_MEM_NORMAL; - else - mem_state |= OBMM_SHM_MEM_NORMAL_NC; - pr_debug("VMA init mem_state: vma_flags=0x%lx, cacheable=%d, mem_state=0x%lx\n", - vm_flags, cacheable, mem_state); - return mem_state; -} - /* VMA operations for obmm-mmaped VMA */ static void obmm_vma_open(struct vm_area_struct *vma) { - pr_debug("VMA opened range (0x%lx-0x%lx)\n", vma->vm_start, vma->vm_end); + struct obmm_region *reg = vma->vm_file->private_data; + + /* atomic to avoid deadlock: update_range holds state_mutex when calling split_vma */ + atomic_inc(®->mmap_count); } static void obmm_vma_close(struct vm_area_struct *vma) { - struct obmm_region *reg; + struct obmm_region *reg = vma->vm_file->private_data; + uint8_t access = vm_flags_to_access(vma->vm_flags); + unsigned long region_pgoff = vma->vm_pgoff; + unsigned long npages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + uint8_t cache_ops; int ret; - reg = (struct obmm_region *)vma->vm_file->private_data; - mutex_lock(®->state_mutex); - /* cc-mmap */ - if (reg->mmap_mode == OBMM_MMAP_NORMAL && reg->ownership_info) { - /* flush cache */ - struct obmm_cmd_update_range update_info = { - .start = vma->vm_start, - .end = vma->vm_end, - .mem_state = OBMM_SHM_MEM_NO_ACCESS, - .cache_ops = OBMM_SHM_CACHE_INFER, - }; - ret = scan_and_flush(reg, vma, &update_info); - if (ret) - pr_err("vma close: failed to flush cache\n"); - remove_mapping_permission(reg, vma, vma->vm_start, vma->vm_end); - release_local_state_info(vma); + cache_ops = update_vma_perm_count(reg, region_pgoff, npages, access, OBMM_SHM_MEM_NO_ACCESS); + + if (cache_ops != OBMM_SHM_CACHE_NONE && reg->mmap_mode == OBMM_MMAP_NORMAL) { + ret = obmm_region_flush_range(reg, region_pgoff << PAGE_SHIFT, + npages << PAGE_SHIFT, cache_ops); + if (ret) + pr_err("vma close: cache flush failed: %d\n", ret); } - reg->mmap_count--; - if (reg->mmap_count == 0) { - /* reset mmap_mode */ + if (atomic_dec_and_test(®->mmap_count)) reg->mmap_mode = OBMM_MMAP_INIT; - } + mutex_unlock(®->state_mutex); - pr_debug("obmm_shmdev munmap: mem_id=%d pid=%d vma=[%#lx, %#lx]\n", reg->regionid, - current->pid, vma->vm_start, vma->vm_end); } -static int obmm_vma_may_split(struct vm_area_struct *vma __always_unused, - unsigned long addr __always_unused) +static int obmm_vma_may_split(struct vm_area_struct *vma, unsigned long addr) { - /* not supported */ - pr_err("VMA may split at 0x%lx (range: 0x%lx-0x%lx), but split not supported\n", addr, - vma->vm_start, vma->vm_end); - return -EOPNOTSUPP; + struct obmm_region *reg = vma->vm_file->private_data; + + if (!obmm_is_aligned(reg, addr)) { + pr_err("mmap split must be aligned: addr=%#lx\n", addr); + return -EINVAL; + } + return 0; } static int obmm_vma_mremap(struct vm_area_struct *vma __always_unused) @@ -110,8 +84,6 @@ static bool validate_update_info(const struct obmm_region *region, const struct obmm_cmd_update_range *update_info, bool cacheable) { - bool valid; - if (!cacheable) { pr_err("Ownership operation is not applicable to o-sync mmap %d.\n", region->regionid); @@ -123,14 +95,22 @@ static bool validate_update_info(const struct obmm_region *region, return false; } - valid = update_info->start < update_info->end && - IS_ALIGNED(update_info->start, PAGE_SIZE) && - IS_ALIGNED(update_info->end, PAGE_SIZE); - if (!valid) - pr_err("{pid=%d, start=%#llx end=%#llx is not a valid page range from memdev %d.\n", - current->pid, update_info->start, update_info->end, region->regionid); - return valid; + if (update_info->start >= update_info->end) { + pr_err("invalid range: start=%#llx end=%#llx\n", + update_info->start, update_info->end); + return false; + } + + if (!obmm_is_aligned(region, update_info->start) || + !obmm_is_aligned(region, update_info->end)) { + pr_err("ownership update must be aligned: pid=%d start=%#llx end=%#llx\n", + current->pid, update_info->start, update_info->end); + return false; + } + + return true; } + static int obmm_vma_mprotect(struct vm_area_struct *vma __always_unused, unsigned long start __always_unused, unsigned long end __always_unused, unsigned long newflags __always_unused) @@ -138,18 +118,127 @@ static int obmm_vma_mprotect(struct vm_area_struct *vma __always_unused, pr_warn("mprotect not supported\n"); return -EOPNOTSUPP; } + static vm_fault_t obmm_vma_fault(struct vm_fault *vmf __always_unused) { pr_warn("Unexpected fault\n"); return VM_FAULT_SIGBUS; } -static int obmm_vma_access(struct vm_area_struct *vma __always_unused, - unsigned long addr __always_unused, void *buf __always_unused, - int len __always_unused, int write __always_unused) + +/* + * Walk the page table for @addr and extract the PFN. + * Handles both PTE entries and PMD leaf (huge page) entries. + * Caller must hold mmap_read_lock (or mmap_write_lock) so that the page + * table entries cannot be torn or freed concurrently. + */ +static int obmm_lookup_pfn(struct mm_struct *mm, unsigned long addr, + unsigned long *pfn) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *ptep; + spinlock_t *ptl; + + pgd = pgd_offset(mm, addr); + if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) + return -EINVAL; + + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d))) + return -EINVAL; + + pud = pud_offset(p4d, addr); + if (pud_none(*pud) || unlikely(pud_bad(*pud))) + return -EINVAL; + + pmd = pmd_offset(pud, addr); + + /* PMD leaf (huge page): extract PFN directly */ + if (pmd_leaf(*pmd)) { + if (!pmd_present(*pmd)) + return -EINVAL; + *pfn = pmd_pfn(*pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); + return 0; + } + + if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) + return -EINVAL; + + /* Regular PTE: follow_pte handles PTE-level lock and mapping */ + if (follow_pte(mm, addr, &ptep, &ptl)) + return -EINVAL; + *pfn = pte_pfn(ptep_get(ptep)); + pte_unmap_unlock(ptep, ptl); + return 0; +} + +/* Custom access handler for ptrace/GDB on PFNMAP VMAs */ +static int obmm_vma_access(struct vm_area_struct *vma, unsigned long addr, + void *buf, int len, int write) { - pr_warn("access not supported\n"); - return -EOPNOTSUPP; + struct obmm_region *reg = vma->vm_file->private_data; + unsigned long pfn, prot; + void *kaddr; + bool is_vmap; + int offset = offset_in_page(addr); + int ret = -EINVAL; + + pr_debug("addr=%#lx len=%d write=%d region=%d type=%s\n", + addr, len, write, reg->regionid, + reg->type == OBMM_EXPORT_REGION ? "export" : "import"); + + if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) + return -EINVAL; + + /* Check permission from vm_flags */ + if (!(vma->vm_flags & VM_READ)) + return -EINVAL; + if (write && !(vma->vm_flags & VM_WRITE)) + return -EINVAL; + + /* Get PFN from page table walk */ + if (obmm_lookup_pfn(vma->vm_mm, addr, &pfn)) + return -EINVAL; + + /* Derive prot from vma->vm_page_prot, adjust for kernel access */ + prot = pgprot_val(vma->vm_page_prot); + prot &= ~(PTE_USER | PTE_NG); + prot |= (PTE_PXN | PTE_UXN); + + /* Map one page */ + len = min(len, (int)(PAGE_SIZE - offset)); + + if (reg->type == OBMM_EXPORT_REGION) { + struct page *page; + + if (!pfn_valid(pfn)) + return -EINVAL; + page = pfn_to_page(pfn); + kaddr = vmap(&page, 1, VM_MAP, __pgprot(prot)); + is_vmap = true; + } else { + resource_size_t phys_addr = (resource_size_t)pfn << PAGE_SHIFT; + + kaddr = (__force void *)ioremap_prot(phys_addr, PAGE_SIZE, prot); + is_vmap = false; + } + if (!kaddr) + return -ENOMEM; + + if (write) + ret = copy_mc_to_kernel(kaddr + offset, buf, len) ? -EFAULT : len; + else + ret = copy_mc_to_kernel(buf, kaddr + offset, len) ? -EFAULT : len; + + if (is_vmap) + vunmap(kaddr); + else + iounmap((__force void __iomem *)kaddr); + return ret; } + static const char *obmm_vma_name(struct vm_area_struct *vma __always_unused) { return "OBMM_SHM"; @@ -249,14 +338,17 @@ static pgprot_t mem_state_to_pgprot(unsigned long mem_state) pgprot_t pgprot; /* initialize pgprot to be normal memory pgprot with certain access rights */ - if ((mem_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READONLY) + if ((mem_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READONLY) { pgprot = PAGE_READONLY; - else if ((mem_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READEXEC) + } else if ((mem_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READEXEC) { pgprot = PAGE_READONLY_EXEC; - else if ((mem_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READWRITE) - pgprot.pgprot = _PAGE_READONLY & ~PTE_RDONLY; - else + } else if ((mem_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READWRITE) { + /* Clear PTE_RDONLY for immediate write access on ARM64 */ + pgprot = PAGE_SHARED; + pgprot.pgprot &= ~PTE_RDONLY; + } else { pgprot = PAGE_NONE; + } /* modify cacheability attribute if necessary */ if ((mem_state & OBMM_SHM_MEM_CACHE_MASK) == OBMM_SHM_MEM_NORMAL_NC) @@ -269,7 +361,7 @@ static pgprot_t mem_state_to_pgprot(unsigned long mem_state) static void print_mmap_param(const struct file *file, const struct vm_area_struct *vma) { - const struct obmm_region *reg = (struct obmm_region *)file->private_data; + const struct obmm_region *reg = (const struct obmm_region *)file->private_data; const char *vm_flags_desc, *f_flags_desc; pr_debug("obmm_shmdev mmap: mem_id=%d pid=%d vma=[%#lx, %#lx] pgoff=%#lx ", reg->regionid, @@ -311,10 +403,18 @@ static int obmm_shm_fops_mmap(struct file *file, struct vm_area_struct *vma) uint8_t mem_state; enum obmm_mmap_mode old_mmap_mode; enum obmm_mmap_granu mmap_granu, init_mmap_granu; + uint8_t access; int ret; bool cacheable, o_sync; print_mmap_param(file, vma); + + if (!(vma->vm_flags & VM_MAYSHARE)) { + pr_err("mmap region %d: MAP_PRIVATE is not supported, use MAP_SHARED\n", + reg->regionid); + return -EINVAL; + } + if (!region_allow_mmap(reg)) { pr_err("mmap region %d: not allow to be mmaped\n", reg->regionid); return -EPERM; @@ -332,11 +432,19 @@ static int obmm_shm_fops_mmap(struct file *file, struct vm_area_struct *vma) if (offset & OBMM_MMAP_FLAG_HUGETLB_PMD) { pr_debug("trying hugepage mmap\n"); offset &= ~OBMM_MMAP_FLAG_HUGETLB_PMD; - if (vma->vm_start % PMD_SIZE || vma->vm_end % PMD_SIZE) { - pr_err("error running huge mmap for not pmd-aligned vma: %#lx-%#lx\n", - vma->vm_start, vma->vm_end); + + if (!IS_ALIGNED(vma->vm_start, PMD_SIZE) || + !IS_ALIGNED(vma->vm_end, PMD_SIZE)) { + pr_err("PMD mmap vma not PMD-aligned: start=%#lx end=%#lx\n", + vma->vm_start, vma->vm_end); return -EINVAL; } + + if (!IS_ALIGNED(offset, PMD_SIZE)) { + pr_err("PMD mmap offset not PMD-aligned: offset=%#lx\n", offset); + return -EINVAL; + } + mmap_granu = OBMM_MMAP_GRANU_PMD; } else { mmap_granu = OBMM_MMAP_GRANU_PAGE; @@ -359,22 +467,11 @@ static int obmm_shm_fops_mmap(struct file *file, struct vm_area_struct *vma) goto err_reset_mmap_granu; } - /* - * VM flags considerations - * Compared to legacy device memory, OBMM memory has many different properties: - * 1. does not have side-effects on access (VM_IO not set) - * 2. may be used for core dump output (VM_DONTDUMP not set) - * On the other hand, OBMM and traditional device memory do have some similarities: - * 3. the mapping cannot be inherited on process fork (VM_DONTCOPY set) for now - * 4. VMA merging and expanding makes no sense (VM_DONTEXPAND set) - * 5. the VMA should not be swapped out (VM_LOCKED set) - * 6. mappable import region does not has struct page; mappable export region haves struct - * page, but cannot work as expected since its kernel linear mapping might be modified - * (VM_PFNMAP set) - */ vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_LOCKED | VM_PFNMAP); - cacheable = o_sync ? false : true; - mem_state = get_vma_mem_state(vma->vm_flags, cacheable); + cacheable = !o_sync; + mem_state = vm_flags_to_mem_state(vma->vm_flags, cacheable); + pr_debug("VMA init mem_state: vma_flags=%#lx, cacheable=%d, mem_state=%#x\n", + vma->vm_flags, cacheable, mem_state); /* initial VMA page prot used by the mapping process -- will be changed later */ vma->vm_page_prot = mem_state_to_pgprot(mem_state); @@ -393,37 +490,19 @@ static int obmm_shm_fops_mmap(struct file *file, struct vm_area_struct *vma) /* cc mmap */ if (reg->mmap_mode == OBMM_MMAP_NORMAL) { - if (mmap_granu == OBMM_MMAP_GRANU_PAGE) { - ret = init_local_state_info(vma, mem_state); - if (ret) { - pr_err("init local state info failed: %pe\n", ERR_PTR(ret)); - goto reset_cur_osync; - } - /* - * initialize region-level ownership info if not done yet. - * once initialized, the OBMM ownership will persist until - * the memdev goes offline - */ - ret = init_ownership_info(reg); - if (ret) - goto err_release_local_state_info; - /* - * after ownership_info initialized, mmap_granu should not be - * reset to OBMM_MMAP_GRANU_NONE. - */ - init_mmap_granu = reg->mmap_granu; - ret = check_mmap_allowed(reg, vma, mem_state); - if (ret) - goto err_release_local_state_info; - } + ret = init_ownership_info(reg); + if (ret) + goto reset_cur_osync; ret = map_obmm_region(vma, reg, mmap_granu); if (ret) { pr_err("Failed to mmap region %d. ret=%pe\n", reg->regionid, ERR_PTR(ret)); - goto err_release_local_state_info; + goto reset_cur_osync; } - if (mmap_granu == OBMM_MMAP_GRANU_PAGE) - add_mapping_permission(reg, vma, mem_state); + + access = vm_flags_to_access(vma->vm_flags); + update_vma_perm_count(reg, vma->vm_pgoff, size >> PAGE_SHIFT, + OBMM_SHM_MEM_NO_ACCESS, access); } else { /* cc-region with nc-mmap(o-sync) */ ret = map_obmm_region(vma, reg, mmap_granu); @@ -432,20 +511,8 @@ static int obmm_shm_fops_mmap(struct file *file, struct vm_area_struct *vma) goto reset_cur_osync; } } - reg->mmap_count++; + atomic_inc(®->mmap_count); mutex_unlock(®->state_mutex); - /* - * since OBMM allows changing protection by pages and we will not split - * VMA in near future. Therefore a mismatch between PTE protection and - * VMA flags is inevitable. Our current approach is to avoid all - * possible faults to change the PTE protection on the fly. Here we - * just set the page protection to the most restrictive one to guard - * against unexpected access. - */ - vma->vm_page_prot = vm_get_page_prot(VM_NONE); - if (!cacheable) - vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); - vm_flags_clear(vma, VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE); vma->vm_ops = &obmm_vm_ops; @@ -454,9 +521,6 @@ static int obmm_shm_fops_mmap(struct file *file, struct vm_area_struct *vma) return 0; -err_release_local_state_info: - if (mmap_granu == OBMM_MMAP_GRANU_PAGE) - release_local_state_info(vma); reset_cur_osync: if (old_mmap_mode == OBMM_MMAP_INIT) reg->mmap_mode = OBMM_MMAP_INIT; @@ -467,9 +531,6 @@ static int obmm_shm_fops_mmap(struct file *file, struct vm_area_struct *vma) return ret; } -/* - * Verify whether mem_state is valid. - */ static bool validate_state(uint8_t mem_state) { if (mem_state & ~(OBMM_SHM_MEM_CACHE_MASK | OBMM_SHM_MEM_ACCESS_MASK)) { @@ -477,265 +538,104 @@ static bool validate_state(uint8_t mem_state) return false; } - /* validate cacheability field */ if ((mem_state & OBMM_SHM_MEM_CACHE_MASK) == OBMM_SHM_MEM_CACHE_RESV) { pr_err("Invalid mem_state: %#x -- reserved cacheability", mem_state); return false; } - /* currently no need to validate access permission field */ if (((mem_state & OBMM_SHM_MEM_ACCESS_MASK) == OBMM_SHM_MEM_READEXEC) && - (((mem_state & OBMM_SHM_MEM_CACHE_MASK) == OBMM_SHM_MEM_DEVICE) || + ((mem_state & OBMM_SHM_MEM_CACHE_MASK) == OBMM_SHM_MEM_DEVICE || (mem_state & OBMM_SHM_MEM_CACHE_MASK) == OBMM_SHM_MEM_NORMAL_NC)) { - pr_err("Bad target mem_state configuration: NC memory cannot be executable\n"); + pr_err("NC memory cannot be executable\n"); return false; } - if (((mem_state & OBMM_SHM_MEM_CACHE_MASK) == OBMM_SHM_MEM_NORMAL_NC) && - ((mem_state & OBMM_SHM_MEM_ACCESS_MASK) != OBMM_SHM_MEM_NO_ACCESS)) { - pr_err("Invalid access state transition: cannot set cacheable region to an accessible but non-cacheable state.\n"); + if ((mem_state & OBMM_SHM_MEM_CACHE_MASK) == OBMM_SHM_MEM_NORMAL_NC && + (mem_state & OBMM_SHM_MEM_ACCESS_MASK) != OBMM_SHM_MEM_NO_ACCESS) { + pr_err("Cannot set cacheable region to accessible but non-cacheable state\n"); return false; } return true; } -static bool validate_cache_ops(uint8_t cache_ops) -{ - if (cache_ops != OBMM_SHM_CACHE_NONE && - cache_ops != OBMM_SHM_CACHE_INVAL && - cache_ops != OBMM_SHM_CACHE_WB_INVAL) { - pr_err("Invalid cache operations: 0x%x\n", cache_ops); - return false; - } - return true; -} - -static int update_pte_prot(pte_t *ptep, unsigned long addr __always_unused, void *data) -{ - pgprot_t *pgprot = (pgprot_t *)data; - pte_t ptent_old, ptent_new; - - ptent_old = ptep_get(ptep); - - ptent_new = pfn_pte(pte_pfn(ptent_old), *pgprot); - if (pte_special(ptent_old)) - ptent_new = pte_mkspecial(ptent_new); - - set_pte(ptep, ptent_new); - return 0; -} - -static void log_ownership_change(struct obmm_region *reg, uint64_t start, uint64_t end, - uint8_t mem_state, uint8_t cache_ops) -{ - pr_debug("obmm memory %d ownership change: pid=%d start=%#llx end=%#llx mem_state=%u cache_ops=%u\n", - reg->regionid, current->pid, start, end, mem_state, cache_ops); -} - -/* the caller holds mm mmap lock */ -static long update_region_page_range(const struct obmm_cmd_update_range *update_info) -{ - int ret; - pgprot_t pgprot; - - /* decide new page protection properties */ - pgprot = mem_state_to_pgprot(update_info->mem_state); - - /* - * we currently do not update VMA properties. Instead we manipulate the - * page table entries directly: VMA-level manipulation is not - * preferrable because the users want to have page-level control. - * Sub-VMA manipulations, which involves frequent merge and split, - * require efforts. But we just do not have enough time. - */ +struct update_prot_info { + pgprot_t newprot; +}; - pr_debug("changing pgtable pgprot to 0x%llx: pid=%d start=0x%llx end=0x%llx\n", - pgprot_val(pgprot), current->pid, update_info->start, update_info->end); - /* not sure whether this part MUST be protected by the write lock */ - ret = apply_to_page_range(current->mm, update_info->start, - update_info->end - update_info->start, update_pte_prot, &pgprot); - if (ret) { - pr_err("failed to change pgprot to 0x%llx: pid=%d start=0x%llx end=0x%llx\n", - pgprot_val(pgprot), current->pid, update_info->start, update_info->end); - return ret; +static int update_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next __always_unused, struct mm_walk *walk) +{ + struct update_prot_info *info = walk->private; + pgprot_t newprot = info->newprot; + struct mm_struct *mm = walk->mm; + spinlock_t *ptl; + pmd_t old_pmd, new_pmd; + + if (pmd_none(*pmd)) + return 0; + + if (pmd_leaf(*pmd)) { + ptl = pmd_lock(mm, pmd); + old_pmd = *pmd; + if (pmd_leaf(old_pmd)) { + new_pmd = pfn_pmd(pmd_pfn(old_pmd), newprot); + new_pmd = pmd_mkspecial(pmd_mkhuge(new_pmd)); + __set_pte((pte_t *)pmd, pmd_pte(new_pmd)); + } + spin_unlock(ptl); + /* Skip PTE-level walk and split_huge_pmd for PFN-mapped huge pages */ + walk->action = ACTION_CONTINUE; + return 0; } - pr_debug("user pgtable updated\n"); - obmm_flush_tlb(current->mm); - pr_debug("TLB flushed\n"); + /* Continue to PTE-level walk */ return 0; } -static bool validate_vma_attrs(struct vm_area_struct *vma, struct file *file, - const struct obmm_cmd_update_range *update_info) +static int update_pte_entry(pte_t *pte, unsigned long addr __always_unused, + unsigned long next __always_unused, struct mm_walk *walk) { - if (!vma) { - pr_err("vma not found for update range: start=%#llx end=%#llx.\n", - update_info->start, update_info->end); - return false; - } - if (vma->vm_file == NULL || file == NULL || - vma->vm_file->private_data != file->private_data) { - pr_err("VA range [%#llx, %#llx) is not a mapping of the target memdev.\n", - update_info->start, update_info->end); - return false; - } - if (update_info->start < vma->vm_start || update_info->end > vma->vm_end) { - pr_err("invalid update range: request [%#llx, %#llx), full range [%#lx, %#lx)\n", - update_info->start, update_info->end, vma->vm_start, vma->vm_end); - return false; - } - return true; -} + struct update_prot_info *info = walk->private; + pgprot_t newprot = info->newprot; + pte_t old_pte, new_pte; -struct scan_context { - struct obmm_region *reg; - struct obmm_local_state_info *local_state_info; - unsigned long vma_start; - uint8_t target_mem_state; - uint8_t range_mem_state; - unsigned long local_page_idx; - unsigned long page_count; -}; + old_pte = ptep_get(pte); + if (pte_none(old_pte)) + return 0; -static int do_scan_region_and_flush(struct scan_context *ctx, unsigned long region_page_idx_start, - unsigned long idx_offset_start, unsigned long idx_offset, - bool is_read) -{ - uint8_t cache_ops; - unsigned long phys_offset, size; + new_pte = pfn_pte(pte_pfn(old_pte), newprot); + if (pte_special(old_pte)) + new_pte = pte_mkspecial(new_pte); - cache_ops = is_read ? OBMM_SHM_CACHE_INVAL : OBMM_SHM_CACHE_WB_INVAL; - phys_offset = (region_page_idx_start + idx_offset_start) << PAGE_SHIFT; - size = (idx_offset - idx_offset_start) << PAGE_SHIFT; - return obmm_region_flush_range(ctx->reg, phys_offset, size, cache_ops); + __set_pte(pte, new_pte); + return 0; } /* - * Scan the global permission count and flush the cache - * for intervals where the read permission count is 1 - * and write permission count is 0. + * Bypass VM_PFNMAP skip in walk_page_test -- OBMM PFN mappings + * have no struct page but do need page table permission updates. */ -static int scan_region_and_flush(struct scan_context *ctx, bool is_read) +static int obmm_walk_test(unsigned long start __always_unused, + unsigned long end __always_unused, struct mm_walk *walk __always_unused) { - unsigned long idx_offset, region_page_idx_start, idx_offset_start; - struct obmm_ownership_info *info; - int ret; - uint32_t state_count, read_count, write_count; - bool start_flag, stop_flag; - - info = ctx->reg->ownership_info; - /* translate to region page idx */ - region_page_idx_start = ctx->local_page_idx + ctx->local_state_info->orig_pgoff; - - idx_offset_start = -1; - for (idx_offset = 0; idx_offset < ctx->page_count; idx_offset++) { - state_count = info->mem_state_arr[region_page_idx_start + idx_offset]; - read_count = GET_R_COUNTER(state_count); - write_count = GET_W_COUNTER(state_count); - - if (is_read) { - start_flag = (write_count == 0 && read_count == 1); - stop_flag = (write_count != 0 || read_count != 1); - } else { - start_flag = (write_count == 1); - stop_flag = (write_count != 1); - } - - if (start_flag && idx_offset_start == -1) { - idx_offset_start = idx_offset; - } else if (stop_flag && idx_offset_start != -1) { - /* flush the range [idx_offset_start, idx_offset) */ - ret = do_scan_region_and_flush(ctx, region_page_idx_start, idx_offset_start, - idx_offset, is_read); - if (ret) - return ret; - idx_offset_start = -1; - } - } - /* check if there is a range not flushed */ - if (idx_offset_start != -1) { - ret = do_scan_region_and_flush(ctx, region_page_idx_start, idx_offset_start, - idx_offset, is_read); - if (ret) - return ret; - } return 0; } -static int do_scan_and_flush(struct scan_context *ctx) +static int update_vma_page_range(struct vm_area_struct *vma, uint8_t mem_state) { - int ret; - uint8_t cache_ops; - unsigned long size, vm_start; - - cache_ops = infer_cache_ops(ctx->range_mem_state, ctx->target_mem_state); - vm_start = ctx->vma_start + (ctx->local_page_idx << PAGE_SHIFT); - size = (unsigned long)ctx->page_count << PAGE_SHIFT; - - log_ownership_change(ctx->reg, vm_start, vm_start + size, ctx->target_mem_state, cache_ops); - if (cache_ops == OBMM_SHM_CACHE_NONE) { - /* ignore none ops */ - ret = 0; - } else if (cache_ops == OBMM_SHM_CACHE_WB_INVAL || cache_ops == OBMM_SHM_CACHE_WB_ONLY) { - /* may need to split and flush */ - ret = scan_region_and_flush(ctx, false); - } else { - /* may need to split and flush */ - ret = scan_region_and_flush(ctx, true); - } - return ret; -} - -/* - * Scan pages in a range and flush pages which are not in use. - * The caller holds region state_mutex lock. - */ -static int scan_and_flush(struct obmm_region *reg, struct vm_area_struct *vma, - const struct obmm_cmd_update_range *update_info) -{ - struct obmm_local_state_info *local_state_info; - int idx_offset, page_count, local_page_idx_start, idx_offset_start; - uint8_t mem_state_start, mem_state; - struct scan_context ctx; - int ret; - - page_count = (update_info->end - update_info->start) >> PAGE_SHIFT; + pgprot_t pgprot = mem_state_to_pgprot(mem_state); + struct update_prot_info info = { .newprot = pgprot }; + struct mm_struct *mm = vma->vm_mm; + unsigned long start = vma->vm_start; + unsigned long end = vma->vm_end; + struct mm_walk_ops walk_ops = { + .test_walk = obmm_walk_test, + .pmd_entry = update_pmd_entry, + .pte_entry = update_pte_entry, + }; - local_state_info = (struct obmm_local_state_info *)vma->vm_private_data; - local_page_idx_start = vma_addr_to_page_idx_local(vma, update_info->start); - - ctx.reg = reg; - ctx.local_state_info = local_state_info; - ctx.vma_start = vma->vm_start; - ctx.target_mem_state = update_info->mem_state; - - idx_offset_start = 0; - mem_state_start = local_state_info->local_mem_state_arr[local_page_idx_start]; - for (idx_offset = 1; idx_offset < page_count; idx_offset++) { - mem_state = - local_state_info->local_mem_state_arr[local_page_idx_start + idx_offset]; - if (mem_state == mem_state_start) - continue; - - ctx.range_mem_state = mem_state_start; - ctx.local_page_idx = local_page_idx_start + idx_offset_start; - ctx.page_count = idx_offset - idx_offset_start; - - ret = do_scan_and_flush(&ctx); - if (ret) - return ret; - - idx_offset_start = idx_offset; - mem_state_start = mem_state; - } - - ctx.range_mem_state = mem_state_start; - ctx.local_page_idx = local_page_idx_start + idx_offset_start; - ctx.page_count = idx_offset - idx_offset_start; - ret = do_scan_and_flush(&ctx); - return ret; + return walk_page_range(mm, start, end, &walk_ops, &info); } static void print_update_param(const struct obmm_cmd_update_range *update_info) @@ -760,125 +660,225 @@ static bool validate_ownership_perm(struct file *file, return validate_perm(file, tmp_vmflags); } +/* Mask of vm_flags that ownership update is allowed to modify */ +#define OBMM_UPDATE_VM_FLAGS_MASK (VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE) + static long obmm_shm_update_range(struct file *file, const struct obmm_cmd_update_range *update_info) { - int ret; - unsigned long phys_offset; - struct obmm_region *reg = (struct obmm_region *)file->private_data; + struct obmm_region *reg = file->private_data; struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - struct obmm_local_state_info *local_state_info; - uint8_t cache_ops; + uint8_t old_access, new_access; + vm_flags_t new_vm_flags, old_vm_flags; + unsigned long cursor, region_pgoff, npages, modified_end; + unsigned long region_offset, length; + uint8_t cache_ops, flush_op, old_mem_state, cache_bits; bool cacheable; + int ret; + + VMA_ITERATOR(vmi, mm, update_info->start); print_update_param(update_info); - if (file->f_flags & O_SYNC) - cacheable = false; - else - cacheable = true; - /* quick validation without VMA info. */ + cacheable = !(file->f_flags & O_SYNC); + if (!validate_update_info(reg, update_info, cacheable)) return -EINVAL; - - if (!validate_ownership_perm(file, update_info)) { - pr_err("The target permission is not allowed for the vma.\n"); + if (!validate_ownership_perm(file, update_info)) return -EPERM; - } - if (!validate_state(update_info->mem_state)) return -EINVAL; + if (update_info->cache_ops != OBMM_SHM_CACHE_INFER && + !obmm_is_valid_cache_ops(update_info->cache_ops)) + return -EINVAL; - if (update_info->cache_ops != OBMM_SHM_CACHE_INFER) { - /* validate cache operations */ - if (!validate_cache_ops(update_info->cache_ops)) - return -EINVAL; - } + new_access = update_info->mem_state & OBMM_SHM_MEM_ACCESS_MASK; + new_vm_flags = access_to_vm_flags(new_access); + cache_bits = update_info->mem_state & OBMM_SHM_MEM_CACHE_MASK; - mmap_read_lock(mm); + mmap_write_lock(mm); + mutex_lock(®->state_mutex); - vma = find_vma(mm, update_info->start); - if (!validate_vma_attrs(vma, file, update_info)) { - ret = -EFAULT; - goto err_unlock; - } + /* + * Phase 1: Single traversal - split, update, and flush per VMA + * + * We process each VMA in a single pass: + * 1. Validate continuity and ownership + * 2. Split VMA at range boundaries if needed + * 3. Update ownership counters and vm_flags + * 4. Execute cache flush for this VMA + * + * Rollback tracks modified_end to know which VMAs need restoration. + */ + modified_end = update_info->start; + cursor = update_info->start; + + for_each_vma_range(vmi, vma, update_info->end) { + /* Check VMA continuity */ + if (vma->vm_start > cursor) { + pr_err("VMA gap detected: expected start=%#lx, actual=%#lx\n", + cursor, vma->vm_start); + ret = -EFAULT; + goto rollback; + } + /* Check VMA belongs to this region */ + if (vma->vm_ops != &obmm_vm_ops || + !vma->vm_file || + vma->vm_file->private_data != reg) { + pr_err("VMA does not belong to this region: vma=[%#lx, %#lx)\n", + vma->vm_start, vma->vm_end); + ret = -EFAULT; + goto rollback; + } - local_state_info = (struct obmm_local_state_info *)vma->vm_private_data; + /* Split at start boundary if needed */ + if (vma->vm_start < cursor) { + ret = split_vma(&vmi, vma, cursor, 1); + if (ret) { + pr_err("Failed to split VMA at start boundary: addr=%#lx, ret=%d\n", + cursor, ret); + goto rollback; + } + } + /* Split at end boundary if needed */ + if (vma->vm_end > update_info->end) { + ret = split_vma(&vmi, vma, update_info->end, 0); + if (ret) { + pr_err("Failed to split VMA at end boundary: addr=%#llx, ret=%d\n", + update_info->end, ret); + goto rollback; + } + } - mutex_lock(®->state_mutex); + /* + * Mark this VMA as being modified BEFORE we modify it. + * This ensures rollback includes this VMA even if subsequent + * operations (counter update, flags change, cache flush) fail. + */ + modified_end = vma->vm_end; + + old_access = vm_flags_to_access(vma->vm_flags); + region_pgoff = vma->vm_pgoff; + npages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + region_offset = region_pgoff << PAGE_SHIFT; + length = npages << PAGE_SHIFT; + + /* + * Temporarily stash vm_flags in vm_private_data for rollback. + * This is safe because OBMM VMAs don't use vm_private_data. + */ + vma->vm_private_data = (void *)(uintptr_t)vma->vm_flags; + + cache_ops = update_vma_perm_count(reg, region_pgoff, npages, + old_access, new_access); + + vm_flags_clear(vma, OBMM_UPDATE_VM_FLAGS_MASK); + vm_flags_set(vma, new_vm_flags & OBMM_UPDATE_VM_FLAGS_MASK); + vma->vm_page_prot = mem_state_to_pgprot(update_info->mem_state); + + /* Determine flush operation for this VMA */ + if (update_info->cache_ops != OBMM_SHM_CACHE_INFER) + flush_op = update_info->cache_ops; /* User-specified */ + else if (cache_ops != OBMM_SHM_CACHE_NONE && + reg->mmap_mode == OBMM_MMAP_NORMAL) + flush_op = cache_ops; /* INFER: per-VMA precise result */ + else + flush_op = OBMM_SHM_CACHE_NONE; + + /* Execute cache flush immediately for this VMA */ + if (flush_op != OBMM_SHM_CACHE_NONE) { + ret = obmm_region_flush_range(reg, region_offset, length, flush_op); + if (ret) + goto rollback; + } - ret = check_modify_ownership_allowed(reg, vma, update_info); - if (ret) { - pr_err("check range (%llx-%llx) ownership failed: %d\n", update_info->start, - update_info->end, ret); - goto err_mutex; - } + /* Update page tables for this VMA */ + ret = update_vma_page_range(vma, update_info->mem_state); + if (ret) { + pr_err("Failed to update page tables: vma=[%#lx, %#lx) mem_state=%#x ret=%d\n", + vma->vm_start, vma->vm_end, update_info->mem_state, ret); + goto rollback; + } - ret = update_region_page_range(update_info); - if (ret) - goto err_mutex; - /* - * If the user specifies a cache operation, we perform the operation - * on the range specified by update_info. Otherwise, - * we dynamically calculate whether the cache operation is needed. - */ - if (update_info->cache_ops != OBMM_SHM_CACHE_INFER) { - cache_ops = update_info->cache_ops; - log_ownership_change(reg, update_info->start, update_info->end, - update_info->mem_state, cache_ops); - /* conditionally flush L3 cache & ub controller packet queue */ - phys_offset = update_info->start - vma->vm_start + - (local_state_info->orig_pgoff << PAGE_SHIFT); - ret = obmm_region_flush_range(reg, phys_offset, - update_info->end - update_info->start, cache_ops); - } else { - ret = scan_and_flush(reg, vma, update_info); + cursor = vma->vm_end; } - if (ret) { - /* original ownership has been lost. */ - pr_err("ownership update: failed to flush cache, ret=%pe. not recoverable.\n", - ERR_PTR(ret)); - ret = -ENOTRECOVERABLE; - goto err_mutex; + /* Check if VMAs cover the entire requested range */ + if (cursor < update_info->end) { + pr_err("VMAs do not cover requested range: expected_end=%#llx actual_end=%#lx\n", + update_info->end, cursor); + ret = -EFAULT; + goto rollback; } - update_ownership(reg, vma, update_info); - mutex_unlock(®->state_mutex); - mmap_read_unlock(mm); + obmm_flush_tlb(mm); - pr_debug("obmm_set_ownership: completed.\n"); + mutex_unlock(®->state_mutex); + mmap_write_unlock(mm); return 0; -err_mutex: +rollback: + pr_debug("Rolling back ownership update: region=%d range=[%#llx, %#llx) modified_end=%#lx\n", + reg->regionid, update_info->start, update_info->end, modified_end); + /* Restore only the VMAs we actually modified (up to modified_end) */ + cursor = update_info->start; + while (cursor < modified_end) { + vma = find_vma(mm, cursor); + if (WARN_ON(!vma || vma->vm_start > cursor)) + break; + old_vm_flags = (vm_flags_t)(uintptr_t)vma->vm_private_data; + old_access = vm_flags_to_access(old_vm_flags); + region_pgoff = vma->vm_pgoff; + npages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + + /* Restore counters */ + update_vma_perm_count(reg, region_pgoff, npages, new_access, old_access); + + /* Restore vm_flags */ + vm_flags_clear(vma, OBMM_UPDATE_VM_FLAGS_MASK); + vm_flags_set(vma, old_vm_flags & OBMM_UPDATE_VM_FLAGS_MASK); + + /* Restore vm_page_prot */ + old_mem_state = old_access | cache_bits; + vma->vm_page_prot = mem_state_to_pgprot(old_mem_state); + + /* + * Restore page tables. + * Note: If failure occurred before page table update (e.g., cache flush + * failed), this operation is redundant but harmless - it simply sets + * the page tables to their original values. + */ + update_vma_page_range(vma, old_mem_state); + + cursor = vma->vm_end; + } + + obmm_flush_tlb(mm); + mutex_unlock(®->state_mutex); -err_unlock: - mmap_read_unlock(mm); + mmap_write_unlock(mm); return ret; } static long obmm_shm_fops_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { - long ret; - switch (cmd) { case OBMM_SHMDEV_UPDATE_RANGE: { struct obmm_cmd_update_range cmd_update_range; - ret = (long)copy_from_user(&cmd_update_range, (void __user *)arg, - sizeof(struct obmm_cmd_update_range)); - if (ret) { - pr_err("failed to load update_range argument"); + if (copy_from_user(&cmd_update_range, (void __user *)arg, + sizeof(cmd_update_range))) { + pr_err("failed to load update_range argument\n"); return -EFAULT; } - ret = obmm_shm_update_range(file, &cmd_update_range); - } break; + return obmm_shm_update_range(file, &cmd_update_range); + } default: - ret = -ENOTTY; + pr_err("unknown ioctl command: %#x\n", cmd); + return -ENOTTY; } - return ret; } const struct file_operations obmm_shm_fops = { .owner = THIS_MODULE, @@ -913,7 +913,7 @@ int obmm_shm_dev_add(struct obmm_region *reg) return -EPERM; } - reg->mmap_count = 0; + atomic_set(®->mmap_count, 0); reg->mmap_mode = OBMM_MMAP_INIT; devt = MKDEV(MAJOR(obmm_devt), reg->regionid); diff --git a/drivers/ub/obmm/ubmempool_allocator.c b/drivers/ub/obmm/ubmempool_allocator.c index da6094288f7b2a467f9759cda9ba6b01e9f784e2..c23dff728e4a9caa48031227ce5c4f8d274bd86e 100644 --- a/drivers/ub/obmm/ubmempool_allocator.c +++ b/drivers/ub/obmm/ubmempool_allocator.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include "obmm_core.h" @@ -62,6 +64,9 @@ static void refill_timeout(struct timer_list *timer) static struct mem_allocator mem_allocators[OBMM_MAX_LOCAL_NUMA_NODES]; +/* Global parent kobject for mempool sysfs entries */ +static struct kobject *mempool_kobj; + static void pool_delay_expand(int nid) { if (is_online_local_node(nid) && mem_allocators[nid].allocator.initialized) { @@ -495,6 +500,10 @@ static int init_mem_allocator_granu(enum allocator_id aid) pr_err("unsupported mem_allocator_granu: %s\n", mem_allocator_granu); return -EINVAL; } + if (!is_power_of_2(__obmm_memseg_size)) { + pr_err("mem_allocator_granu must be power of 2: %s\n", mem_allocator_granu); + return -EINVAL; + } if (aid == ALLOCATOR_HUGETLB_PUD && __obmm_memseg_size != PUD_SIZE) { pr_err("unsupported mem_allocator_granu for hugetlb_pud allocator: %s\n", mem_allocator_granu); @@ -518,7 +527,7 @@ static int mem_allocator_init_one(int nid, enum allocator_id aid) timer_setup(&mem_allocators[nid].refill_timer, refill_timeout, 0); ret = conti_mem_allocator_init(allocator, nid, OBMM_MEMSEG_SIZE, allocator_ops[aid], - "%s/%d", allocator_names[aid], nid); + mempool_kobj, "obmm-%d", nid); if (ret) goto err_del_timer; @@ -613,7 +622,6 @@ int ubmempool_allocator_init(void) for_each_online_local_node(nid) { if (nid >= OBMM_MAX_LOCAL_NUMA_NODES) { - /* be no mem_allocators[nid] is out of range */ pr_err("Too many local NUMA nodes. OBMM rebuild required.\n"); return -EOPNOTSUPP; } @@ -637,6 +645,12 @@ int ubmempool_allocator_init(void) if (ret) return ret; + mempool_kobj = kobject_create_and_add("obmm_mempool", kernel_kobj); + if (!mempool_kobj) { + pr_err("Failed to create mempool sysfs directory\n"); + return -ENOMEM; + } + for_each_online_local_node(i) { ret = mem_allocator_init_one(i, aid); if (ret) @@ -652,6 +666,8 @@ int ubmempool_allocator_init(void) if (j < i) mem_allocator_uninit_one(j); } + kobject_put(mempool_kobj); + mempool_kobj = NULL; return ret; } @@ -668,4 +684,9 @@ void ubmempool_allocator_exit(void) mem_allocator_uninit_one(i); } + + if (mempool_kobj) { + kobject_put(mempool_kobj); + mempool_kobj = NULL; + } } diff --git a/drivers/ub/sentry/sentry_remote_client.c b/drivers/ub/sentry/sentry_remote_client.c index ed4ac35cc718b432cf32d0fc77c3ef92eb52e658..3fc042c426a69f7df61ea6284a0f218243554536 100644 --- a/drivers/ub/sentry/sentry_remote_client.c +++ b/drivers/ub/sentry/sentry_remote_client.c @@ -32,7 +32,7 @@ #define PANIC_TIMEOUT_MS_MAX 3600000 #define KERNEL_REBOOT_TIMEOUT_MS_MIN 0 #define KERNEL_REBOOT_TIMEOUT_MS_MAX 3600000 -#define LOCAL_EID_MAX_LEN (EID_MAX_LEN * 2 + 1 + 1) +#define LOCAL_EID_MAX_LEN (EID_MAX_LEN * MAX_DIE_NUM + 1 + 1) #undef pr_fmt #define pr_fmt(fmt) "[sentry][remote client]: " fmt @@ -144,7 +144,7 @@ int remote_event_handler(enum sentry_msg_helper_msg_type remote_type, bool uvb_send_success = false; bool urma_send_success = false; enum sentry_msg_helper_msg_type remote_ack_type; - char send_data[MAX_DIE_NUM][URMA_SEND_DATA_MAX_LEN]; + char send_data[MAX_DIE_NUM][URMA_SEND_DATA_MAX_LEN] = {0}; uint64_t start_count, current_count; uint64_t code_run_count, code_run_times_ms; uint64_t counts_per_sec = arch_timer_get_cntfrq(); @@ -805,8 +805,8 @@ static ssize_t proc_reporter_eid_write(struct file *file, const char __user *ubu { int ret; int eid_num = 0; - char eid_str_buf[LOCAL_EID_MAX_LEN]; - char eid_str_buf_tmp[LOCAL_EID_MAX_LEN]; + char eid_str_buf[LOCAL_EID_MAX_LEN] = {0}; + char eid_str_buf_tmp[LOCAL_EID_MAX_LEN] = {0}; char eid_str_array[MAX_DIE_NUM][EID_MAX_LEN] = {0}; union ubcore_eid eid_ub_buf[MAX_DIE_NUM] = {0}; @@ -1035,14 +1035,15 @@ static int __init sentry_remote_reporter_init(void) if (ret) return ret; - sentry_client_ctx.msg_str = kzalloc(MAX_NODE_NUM * 2 * sizeof(char *), GFP_KERNEL); + sentry_client_ctx.msg_str = kzalloc(MAX_NODE_NUM * MAX_DIE_NUM * sizeof(char *), + GFP_KERNEL); if (!sentry_client_ctx.msg_str) { pr_err("Failed to allocate memory for msg_str\n"); ret = -ENOMEM; goto stop_kthread; } - for (i = 0; i < MAX_NODE_NUM * 2; i++) { + for (i = 0; i < MAX_NODE_NUM * MAX_DIE_NUM; i++) { sentry_client_ctx.msg_str[i] = kzalloc(URMA_SEND_DATA_MAX_LEN, GFP_KERNEL); if (!sentry_client_ctx.msg_str[i]) { pr_err("Failed to allocate memory for msg_str[%d]\n", i); @@ -1079,7 +1080,7 @@ static int __init sentry_remote_reporter_init(void) unregister_kernel_reboot: unregister_reboot_notifier(&kernel_reboot_notifier); free_msg_str: - free_char_array(sentry_client_ctx.msg_str, MAX_NODE_NUM * 2); + free_char_array(sentry_client_ctx.msg_str, MAX_NODE_NUM * MAX_DIE_NUM); stop_kthread: sentry_panic_reporter_exit(); return ret; @@ -1099,7 +1100,7 @@ static void __exit sentry_remote_reporter_exit(void) unregister_reboot_notifier(&kernel_reboot_notifier); pr_info("Kernel reboot handler unregistered\n"); - free_char_array(sentry_client_ctx.msg_str, MAX_NODE_NUM * 2); + free_char_array(sentry_client_ctx.msg_str, MAX_NODE_NUM * MAX_DIE_NUM); if (sentry_client_ctx.panic_proc_dir) proc_remove(sentry_client_ctx.panic_proc_dir); diff --git a/drivers/ub/sentry/sentry_remote_server.c b/drivers/ub/sentry/sentry_remote_server.c index 42b24a3a0bd916008723688163d51674b0e1a707..135842d180736658b28d1a7e383916957825a1f7 100644 --- a/drivers/ub/sentry/sentry_remote_server.c +++ b/drivers/ub/sentry/sentry_remote_server.c @@ -135,7 +135,7 @@ int send_msg_to_userspace_and_ack(struct sentry_msg_helper_msg *msg, } /* Get acknowledgment success, send acknowledgment message */ - char send_ack[URMA_SEND_DATA_MAX_LEN]; + char send_ack[URMA_SEND_DATA_MAX_LEN] = {0}; ret = snprintf(send_ack, URMA_SEND_DATA_MAX_LEN, "%d_%u_%s_%lu", ack_type, @@ -274,7 +274,7 @@ int create_kthread_to_process_msg(const char *event_msg, enum SENTRY_REMOTE_COMM_TYPE comm_type) { int ret; - struct sentry_msg_helper_msg msg; + struct sentry_msg_helper_msg msg = {0}; uint32_t random_id; struct child_thread_process_data *child_data; struct task_struct *child_thread; @@ -388,7 +388,7 @@ static int process_urma_data(void *data) } free_msg: - free_char_array(msg_str, MAX_NODE_NUM); + free_char_array(msg_str, MAX_NODE_NUM * MAX_DIE_NUM); pr_info("Urma receiver thread stopped!\n"); return ret; diff --git a/drivers/ub/sentry/sentry_reporter.c b/drivers/ub/sentry/sentry_reporter.c index 76ebb3d86b6ffca5f1e1ea4856e9b65126a966bc..16a57825f80889f408ebc05261ff52099f5d0231 100644 --- a/drivers/ub/sentry/sentry_reporter.c +++ b/drivers/ub/sentry/sentry_reporter.c @@ -114,7 +114,7 @@ static int acpi_power_notifier_callback(struct notifier_block *nb, unsigned long action, void *data) { int ret; - struct sentry_msg_helper_msg msg; + struct sentry_msg_helper_msg msg = {0}; if (!g_power_off_enable) return NOTIFY_OK; @@ -147,7 +147,7 @@ static int lowmem_notifier_callback(struct notifier_block *nb, unsigned long action, void *parm) { struct reclaim_notify_data *data = parm; - struct sentry_msg_helper_msg msg; + struct sentry_msg_helper_msg msg = {0}; int ret; int i; @@ -463,7 +463,7 @@ static const struct proc_ops proc_ub_mem_fault_with_kill_file_operations = { */ static int ub_mem_ras_handler(uint64_t phys_addr, enum ras_err_type err_type) { - struct sentry_msg_helper_msg msg; + struct sentry_msg_helper_msg msg = {0}; struct page *page; int ret; diff --git a/drivers/ub/sentry/sentry_urma_comm.c b/drivers/ub/sentry/sentry_urma_comm.c index fcc55b29726eec9d2fddbd6acfa768fafca52926..f50a11ae9af734e90a3f0a77af59e0ee661bc606 100644 --- a/drivers/ub/sentry/sentry_urma_comm.c +++ b/drivers/ub/sentry/sentry_urma_comm.c @@ -566,7 +566,7 @@ static void release_all_resource(void) */ int str_to_eid(const char *eid_str, union ubcore_eid *eid) { - if (strlen(eid_str) != EID_MAX_LEN - 1) { + if (!eid_str || !eid || strlen(eid_str) != EID_MAX_LEN - 1) { pr_err("eid str %s len is invalid, failed to transfer\n", eid_str); return -EINVAL; } @@ -1084,6 +1084,11 @@ int match_index_by_remote_ub_eid(union ubcore_eid remote_eid, int *node_index, i { int i, j; + if (!node_index || !die_index) { + pr_err("Invalid param, failed to match index\n"); + return -EINVAL; + } + for (i = 0; i < sentry_urma_ctx.local_eid_num_configured; i++) { if (!sentry_urma_dev[i].is_created) { pr_err("invalid value for sentry_urma_dev[%d].is_created\n", i); @@ -1123,8 +1128,7 @@ EXPORT_SYMBOL(match_index_by_remote_ub_eid); */ int sentry_create_urma_resource(union ubcore_eid eid[], int eid_num) { - int ret; - int i; + int ret, i; /* Prepare for new device matching by cleaning up old resources */ release_all_resource(); @@ -1227,6 +1231,11 @@ int process_multi_eid_string(char *eid_buf, char eid_array[][EID_MAX_LEN], int eid_num = 0; char *eid_part; + if (!eid_buf || !sepstr) { + pr_err("Invalid param, failed to process multi eid string\n"); + return -EINVAL; + } + while ((eid_part = strsep(&eid_buf, sepstr)) != NULL) { if (eid_num >= eid_max_num) { pr_err("Invalid eid format: max num %d, current input exceeds\n", @@ -1234,7 +1243,7 @@ int process_multi_eid_string(char *eid_buf, char eid_array[][EID_MAX_LEN], return -EINVAL; } - if (strlen(eid_part) > EID_MAX_LEN) { + if (strlen(eid_part) >= EID_MAX_LEN) { pr_err("Invalid eid format: str too long: %s\n", eid_part); return -EINVAL; } @@ -1245,7 +1254,8 @@ int process_multi_eid_string(char *eid_buf, char eid_array[][EID_MAX_LEN], return -EINVAL; } - memcpy(eid_array[eid_num], eid_part, EID_MAX_LEN); + memset(eid_array[eid_num], 0, EID_MAX_LEN); + memcpy(eid_array[eid_num], eid_part, strnlen(eid_part, EID_MAX_LEN)); eid_num++; } @@ -2070,6 +2080,12 @@ int urma_send(const char *buf, size_t len, const char *dst_eid, int die_index) if (!g_is_created_ubcore_resource) return -ENODEV; + /* at broadcast mode, dst_eid is NULL */ + if (!buf || len > URMA_SEND_DATA_MAX_LEN || strlen(buf) > len) { + pr_err("%s: Invalid param, failed to send data\n", __func__); + return -EINVAL; + } + if (!dst_eid && die_index >= 0) { /* Broadcast mode: send to all nodes */ cnt = urma_send_to_all_nodes(buf, len, die_index); @@ -2146,8 +2162,8 @@ int urma_recv(char **buf_arr, size_t len) int tmp_die_index = die_index; /* Extract message from completion context */ - ret = snprintf(recv_msg, len, "%s", (char *)sentry_urma_ctx.urma_recv_cr[i].user_ctx); - if ((size_t)ret >= len) { + ret = snprintf(recv_msg, sizeof(recv_msg), "%s", (char *)sentry_urma_ctx.urma_recv_cr[i].user_ctx); + if ((size_t)ret >= sizeof(recv_msg)) { pr_warn("urma recv: msg size exceeds max len %lu\n", len); continue; } @@ -2250,7 +2266,7 @@ static int __init sentry_urma_comm_init(void) { int ret = 0; - sentry_urma_ctx.proc_dir = proc_mkdir_mode(PROC_DEVICE_PATH, 0550, NULL); + sentry_urma_ctx.proc_dir = proc_mkdir_mode(PROC_DEVICE_PATH, PROC_DIR_PERMISSION, NULL); if (!sentry_urma_ctx.proc_dir) { pr_err("create /proc/%s dir failed\n", PROC_DEVICE_PATH); return -ENOMEM; diff --git a/drivers/ub/sentry/sentry_uvb_comm.c b/drivers/ub/sentry/sentry_uvb_comm.c index e7fef72e0670ae3f8654efec44af67740d603d0a..9f6d9038142d4586258ba5e0c8e22bc4bbf431b5 100644 --- a/drivers/ub/sentry/sentry_uvb_comm.c +++ b/drivers/ub/sentry/sentry_uvb_comm.c @@ -45,7 +45,12 @@ int uvb_send(const char *str, uint32_t dst_cna, bool is_sync) { int res, cnt = 0; - struct cis_message msg; + if (!str) { + pr_err("%s: Invalid param, failed to send data\n", __func__); + return -EINVAL; + } + + struct cis_message msg = {0}; msg.input = (char *)str; msg.input_size = strlen(str) + 1; @@ -86,10 +91,15 @@ EXPORT_SYMBOL(uvb_send); static int convert_server_cna_str_to_u32_array(const char *server_cna) { int server_cna_valid_num = 0, ret = 0; - uint32_t server_cna_array[MAX_NODE_NUM]; + uint32_t server_cna_array[MAX_NODE_NUM] = {0}; char *token; char *server_cna_copy = kstrdup(server_cna, GFP_KERNEL); + if (!server_cna_copy) { + pr_err("%s: kstrdup failed!\n", __func__); + return -ENOMEM; + } + char *rest = server_cna_copy; while ((token = strsep(&rest, ";"))) { @@ -134,7 +144,7 @@ static ssize_t proc_uvb_server_cna_write(struct file *file, const char __user *u size_t count, loff_t *ppos) { int ret = 0; - char server_cna_buf[(MAX_NODE_NUM + 1) * INTEGER_TO_STR_MAX_LEN]; + char server_cna_buf[(MAX_NODE_NUM + 1) * INTEGER_TO_STR_MAX_LEN] = {0}; if (count > (MAX_NODE_NUM + 1) * INTEGER_TO_STR_MAX_LEN - 1) { pr_err("invalid value for server_cna mode.\n"); diff --git a/drivers/ub/sentry/smh_common_type.h b/drivers/ub/sentry/smh_common_type.h index 069c04a23f0876a1081ae3a12de2dd4caf7bb33c..1f50bc05d9ac37e9c9ce52721ea22fa79ac94a44 100644 --- a/drivers/ub/sentry/smh_common_type.h +++ b/drivers/ub/sentry/smh_common_type.h @@ -15,6 +15,7 @@ #include #include #include +#include #define SMH_TYPE ('}') #define MAX_DIE_NUM 2 @@ -103,7 +104,12 @@ static inline int convert_str_to_smh_msg(const char *str, { int n; int ret = 0; - char input_copy[URMA_SEND_DATA_MAX_LEN]; + char input_copy[URMA_SEND_DATA_MAX_LEN + 1] = {0}; + + if (!str || strnlen(str, URMA_SEND_DATA_MAX_LEN + 1) > URMA_SEND_DATA_MAX_LEN) { + pr_err("%s: Invalid string, failed to convert it to smh_msg.\n", __func__); + return -EINVAL; + } n = sscanf(str, "%d_%s", (int *)&smh_msg->type, input_copy); if (n != 2) { diff --git a/drivers/ub/sentry/smh_core.c b/drivers/ub/sentry/smh_core.c index 61103c551228da6106abfb7de64badb119ed4077..a4dbf010012b4da6de6f781e00dbb95aacde9445 100644 --- a/drivers/ub/sentry/smh_core.c +++ b/drivers/ub/sentry/smh_core.c @@ -34,7 +34,7 @@ static long smh_dev_ioctl(struct file *file, unsigned int cmd, unsigned long arg switch (cmd) { case SMH_MSG_ACK: { - struct sentry_msg_helper_msg cmd_msg; + struct sentry_msg_helper_msg cmd_msg = {0}; if (copy_from_user(&cmd_msg, (void __user *)arg, sizeof(cmd_msg))) return -EFAULT; diff --git a/drivers/ub/sentry/smh_message.c b/drivers/ub/sentry/smh_message.c index 971e659244d4c38126d6cca724b073933ebe63ca..7e0e0c319715df39cc87e6bec91228067c14bebd 100644 --- a/drivers/ub/sentry/smh_message.c +++ b/drivers/ub/sentry/smh_message.c @@ -145,6 +145,11 @@ int smh_message_send(struct sentry_msg_helper_msg *msg, bool ack) int ret = 0; struct smh_msg_handler *handle; + if (!msg) { + pr_err("%s: Invalid param, failed to send data\n", __func__); + return -EINVAL; + } + if (!msg->msgid) { RM_LOG_ERR("please set the correct msgid by 'smh_get_new_msg_id', stop to send this msg\n"); return -EINVAL; @@ -304,6 +309,11 @@ int smh_message_get_ack(struct sentry_msg_helper_msg *msg) struct smh_msg_handler *handle; bool found = false; + if (!msg) { + pr_err("%s: Invalid param, failed to get data\n", __func__); + return found; + } + FIND_AND_REMOVE_TIMEOUT_FROM_LIST(handle, &msg_ctx.msgbuf_get_lock, &msg_ctx.msgbuf_get, get_list, msg->msgid, found); @@ -358,6 +368,7 @@ int smh_message_init(void) */ void smh_message_exit(void) { + int ret; struct smh_msg_handler *handle, *tmp; /* Clean up acknowledgment list */ @@ -376,5 +387,14 @@ void smh_message_exit(void) } spin_unlock(&msg_ctx.msgbuf_get_lock); + do { + ret = kfifo_out_spinlocked(&msg_ctx.msgbuf_send, &handle, + sizeof(handle), &msg_ctx.msgbuf_send_lock); + if (ret) { + kfree(handle); + continue; + } + break; + } while (1); kfifo_free(&msg_ctx.msgbuf_send); } diff --git a/include/linux/mm.h b/include/linux/mm.h index 9e4410ea290b8e5f3abae7af69a8fdc3163969ef..93f3fcdbdf5eebe09cc3fcc2b6b69a3bdd58ca6a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3421,6 +3421,8 @@ extern int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma, extern int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, unsigned long start, unsigned long end, pgoff_t pgoff); extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); +extern int split_vma(struct vma_iterator *vmi, struct vm_area_struct *, + unsigned long addr, int new_below); extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); extern void unlink_file_vma(struct vm_area_struct *); extern struct vm_area_struct *copy_vma(struct vm_area_struct **, diff --git a/mm/mmap.c b/mm/mmap.c index 93c1c1621309dba6270749b9e20550dbd10a0641..69695ec7b981176f889a8a30e4b67d1408c11266 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2500,7 +2500,7 @@ static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, * Split a vma into two pieces at address 'addr', a new vma is allocated * either for the first part or the tail. */ -static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, +int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, unsigned long addr, int new_below) { #ifdef CONFIG_PID_NS @@ -2512,6 +2512,7 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, return __split_vma(vmi, vma, addr, new_below); } +EXPORT_SYMBOL(split_vma); /* * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 3d26a7ef74de2bc5e1abee7b5d7b7e5944489791..4f5192adf27e9fc19c9ba4bf474f849c39edf2f9 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1624,7 +1624,7 @@ static void kvm_replace_memslot(struct kvm *kvm, * only allows these. */ #define KVM_SET_USER_MEMORY_REGION_V1_FLAGS \ - (KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_READONLY) + (KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_READONLY | KVM_MEM_HUGE_POD) static int check_memory_region_flags(struct kvm *kvm, const struct kvm_userspace_memory_region2 *mem)