diff --git a/0001-LU-15658-lod-ost-list-and-pool-name-conflict.patch b/0001-LU-15658-lod-ost-list-and-pool-name-conflict.patch new file mode 100644 index 0000000000000000000000000000000000000000..0859f08cf18b2616422c925072ecb416e608d8f8 --- /dev/null +++ b/0001-LU-15658-lod-ost-list-and-pool-name-conflict.patch @@ -0,0 +1,150 @@ +From b4724968d1b40233445e4e1f821d1c5efc409d4b Mon Sep 17 00:00:00 2001 +From: Vitaly Fertman +Date: Wed, 30 Mar 2022 22:11:42 +0300 +Subject: [PATCH 01/49] LU-15658 lod: ost list and pool name conflict + +If the OST list is given on setstripe with the -o option, the pool is +unconditionally dropped even if all the OSTs are in. Let the pool stay +in this case. + +Also, if the start index given on setstripe with the -i option is out +of the pool, make it similar to the -o option - drop the pool. + +lustre-change: https://review.whamcloud.com/46967 +Lustre-commit: 06dd5a4638dd36640b146d4388c09a322873760b + +HPE-bug-id: LUS-10868 +Fixes: b384ea39e5 ("LU-14480 pool: wrong usage with ost list") +Signed-off-by: Vitaly Fertman +Change-Id: I718c237e273689048eb74044eea73de6c212395e +Reviewed-by: Andreas Dilger +Reviewed-by: Etienne AUJAMES +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51285 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + lustre/lod/lod_qos.c | 86 +++++++++++++++++++++++++++----------------- + 1 file changed, 54 insertions(+), 32 deletions(-) + +diff --git a/lustre/lod/lod_qos.c b/lustre/lod/lod_qos.c +index 5d7af0c2c0..0cb71d05c8 100644 +--- a/lustre/lod/lod_qos.c ++++ b/lustre/lod/lod_qos.c +@@ -2140,6 +2140,59 @@ unlock: + RETURN(rc); + } + ++static void lod_qos_set_pool(struct lod_object *lo, int pos, char *pool_name, ++ struct lov_user_md_v1 *v1) ++{ ++ struct lod_device *d = lu2lod_dev(lod2lu_obj(lo)->lo_dev); ++ struct lod_layout_component *lod_comp; ++ struct pool_desc *pool = NULL; ++ int j, rc = 0; ++ ++ /* In the function below, .hs_keycmp resolves to ++ * pool_hashkey_keycmp() */ ++ /* coverity[overrun-buffer-val] */ ++ if (pool_name) ++ pool = lod_find_pool(d, pool_name); ++ ++ if (pool) { ++ lod_comp = &lo->ldo_comp_entries[pos]; ++ if (lod_comp->llc_stripe_offset != LOV_OFFSET_DEFAULT) { ++ if (v1->lmm_magic == LOV_USER_MAGIC_SPECIFIC) { ++ struct lov_user_md_v3 *v3; ++ ++ v3 = (struct lov_user_md_v3 *)v1; ++ for (j = 0; j < v3->lmm_stripe_count; j++) { ++ __u32 num; ++ ++ num = lod_comp->llc_ostlist.op_array[j]; ++ rc = lod_check_index_in_pool(num, pool); ++ if (rc) ++ break; ++ } ++ } else { ++ rc = lod_check_index_in_pool( ++ lod_comp->llc_stripe_offset, pool); ++ } ++ if (rc < 0) { ++ CDEBUG(D_LAYOUT, "%s: index %u is not in the " ++ "pool %s, dropping the pool\n", ++ lod2obd(d)->obd_name, ++ lod_comp->llc_stripe_offset, ++ pool_name); ++ pool_name = NULL; ++ } ++ } ++ ++ if (lod_comp->llc_stripe_count > pool_tgt_count(pool) && ++ !(lod_comp->llc_pattern & LOV_PATTERN_OVERSTRIPING)) ++ lod_comp->llc_stripe_count = pool_tgt_count(pool); ++ ++ lod_pool_putref(pool); ++ } ++ ++ lod_obj_set_pool(lo, pos, pool_name); ++} ++ + /** + * Parse suggested striping configuration. + * +@@ -2276,7 +2329,6 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, + LASSERT(lo->ldo_comp_entries); + + for (i = 0; i < comp_cnt; i++) { +- struct pool_desc *pool; + struct lu_extent *ext; + char *pool_name; + +@@ -2310,8 +2362,6 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, + rc = lod_comp_copy_ost_lists(lod_comp, v3); + if (rc) + GOTO(free_comp, rc); +- +- pool_name = NULL; + } + } + +@@ -2344,35 +2394,7 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, + } + + lod_comp->llc_stripe_offset = v1->lmm_stripe_offset; +- lod_obj_set_pool(lo, i, pool_name); +- +- if (pool_name == NULL) +- continue; +- +- /* In the function below, .hs_keycmp resolves to +- * pool_hashkey_keycmp() */ +- /* coverity[overrun-buffer-val] */ +- pool = lod_find_pool(d, pool_name); +- if (pool == NULL) +- continue; +- +- if (lod_comp->llc_stripe_offset != LOV_OFFSET_DEFAULT) { +- rc = lod_check_index_in_pool( +- lod_comp->llc_stripe_offset, pool); +- if (rc < 0) { +- lod_pool_putref(pool); +- CDEBUG(D_LAYOUT, "%s: invalid offset, %u\n", +- lod2obd(d)->obd_name, +- lod_comp->llc_stripe_offset); +- GOTO(free_comp, rc = -EINVAL); +- } +- } +- +- if (lod_comp->llc_stripe_count > pool_tgt_count(pool) && +- !(lod_comp->llc_pattern & LOV_PATTERN_OVERSTRIPING)) +- lod_comp->llc_stripe_count = pool_tgt_count(pool); +- +- lod_pool_putref(pool); ++ lod_qos_set_pool(lo, i, pool_name, v1); + } + + RETURN(0); +-- +2.33.0 + diff --git a/0001-Prepare-for-next-pointrelease.patch b/0001-Prepare-for-next-pointrelease.patch deleted file mode 100644 index c9b24c11616cb3e4a16fd3434ee32aec8a27d00e..0000000000000000000000000000000000000000 --- a/0001-Prepare-for-next-pointrelease.patch +++ /dev/null @@ -1,97 +0,0 @@ -From c209fe8c68e4275855154285901cfc0091bdf9e9 Mon Sep 17 00:00:00 2001 -From: Oleg Drokin -Date: Tue, 18 Jul 2023 10:54:54 -0400 -Subject: [PATCH 01/61] Prepare for next pointrelease. - -Change-Id: Idc2b25dea2b4c0f587f735dc1bdf2dd358d1f647 ---- - lustre/ChangeLog | 76 +++++++++++++++++++++++++++++++++++++++++++++++- - 1 file changed, 75 insertions(+), 1 deletion(-) - -diff --git a/lustre/ChangeLog b/lustre/ChangeLog -index 59b05d21f2..57699d2232 100644 ---- a/lustre/ChangeLog -+++ b/lustre/ChangeLog -@@ -1,4 +1,78 @@ --06-19-2023 Whamcloud -+TBD Whamcloud -+ * version 2.15.4 -+ * See https://wiki.whamcloud.com/display/PUB/Lustre+Support+Matrix -+ for currently supported client and server kernel versions. -+ * Server primary kernels built and tested during release cycle: -+ 4.18.0-477.10.1.el8 (RHEL8.8) -+ * Other server kernels known to build and work at some point (others may also work): -+ 3.10.0-862.14.4.el7 (RHEL7.5) -+ 3.10.0-957.27.2.el7 (RHEL7.6) -+ 3.10.0-1062.18.1.el7 (RHEL7.7) -+ 3.10.0-1127.19.1.el7 (RHEL7.8) -+ 3.10.0-1160.88.1.el7 (RHEL7.9) -+ 4.18.0-80.11.2.el8 (RHEL8.0) -+ 4.18.0-147.8.1.el8 (RHEL8.1) -+ 4.18.0-193.28.1.el8 (RHEL8.2) -+ 4.18.0-240.22.1.el8 (RHEL8.3) -+ 4.18.0-305.25.1.el8 (RHEL8.4) -+ 4.18.0-348.23.1.el8 (RHEL8.5) -+ 4.18.0-372.32.1.el8 (RHEL8.6) -+ 4.18.0-425.3.1.el8 (RHEL8.7) -+ 4.4.120-92.70 (SLES12 SP2) -+ 4.4.180-94.100 (SLES12 SP3) -+ 4.4.0-131 (Ubuntu 16.04) -+ 4.15.0-32 (Ubuntu 18.04) -+ 5.4.0-48 (Ubuntu 20.04) -+ vanilla linux 5.4.0 (ZFS + ldiskfs) -+ vanilla linux 5.4.21 (ZFS + ldiskfs) -+ vanilla linux 5.4.136 (ZFS + ldiskfs) -+ * ldiskfs needs an ldiskfs patch series for that kernel, ZFS does not -+ * Client primary kernels built and tested during release cycle: -+ 5.14.0-284.11.1.el9 (RHEL9.2) -+ 5.14.0-162.23.1.el9 (RHEL9.1) -+ 4.18.0-477.10.1.el8 (RHEL8.8) -+ 5.4.0-37 (Ubuntu 20.04) -+ 5.14.21-150400.24.28 (SLES15 SP4) -+ * Other clients known to build on these kernels at some point (others may also work): -+ 3.10.0-862.14.4.el7 (RHEL7.5) -+ 3.10.0-957.27.2.el7 (RHEL7.6) -+ 4.14.0-49.13.1.el7a (RHEL7.5) -+ 4.14.0-115.2.2.el7a (RHEL7.6) -+ 3.10.0-1062.18.1.el7 (RHEL7.7) -+ 3.10.0-1127.19.1.el7 (RHEL7.8) -+ 3.10.0-1160.88.1.el7 (RHEL7.9) -+ 4.18.0-80.11.2.el8 (RHEL8.0) -+ 4.18.0-147.8.1.el8 (RHEL8.1) -+ 4.18.0-193.28.1.el8 (RHEL8.2) -+ 4.18.0-240.22.1.el8 (RHEL8.3) -+ 4.18.0-305.25.1.el8 (RHEL8.4) -+ 4.18.0-348.23.1.el8 (RHEL8.5) -+ 4.18.0-372.32.1.el8 (RHEL8.6) -+ 4.18.0-425.3.1.el8 (RHEL8.7) -+ 5.14.0-70.30.1.el9 (RHEL9.0) -+ 4.4.120-92.70 (SLES12 SP2) -+ 4.4.180-94.100 (SLES12 SP3) -+ 4.12.14-95.48 (SLES12 SP4) -+ 4.12.14-122.91 (SLES12 SP5) -+ 4.12.14-197.75 (SLES15 SP1) -+ 5.3.18-24.96 (SLES15 SP2) -+ 5.3.18-150300.59.93 (SLES15 SP3) -+ 4.4.0-131 (Ubuntu 16.04) -+ 4.15.0-48 (Ubuntu 18.04) -+ 5.8.0-53 (Ubuntu 20.04.2 HWE) -+ 5.11.0-31 (Ubuntu 20.04.3 HWE) -+ 5.11.0 (vanilla kernel.org) -+ * Recommended e2fsprogs version: 1.47.0-wc1 or newer -+ * Recommended ZFS version: 2.1.11 -+ * NFS export disabled when stack size < 8192 (32-bit Lustre clients), -+ since the NFSv4 export of Lustre filesystem with 4K stack may cause a -+ stack overflow. For more information, please refer to bugzilla 17630. -+ * NFSv4 reexport to 32-bit NFS client nodes requires Lustre client on -+ the re-exporting nodes to be mounted with "32bitapi" mount option -+ -+-------------------------------------------------------------------------------- -+ -+'06-19-2023 Whamcloud - * version 2.15.3 - * See https://wiki.whamcloud.com/display/PUB/Lustre+Support+Matrix - for currently supported client and server kernel versions. --- -2.33.0 - diff --git a/0002-LU-15821-ldlm-Prioritize-blocking-callbacks.patch b/0002-LU-15821-ldlm-Prioritize-blocking-callbacks.patch deleted file mode 100644 index 1cdbf0de16d89a85477b05f23f295a7daa663496..0000000000000000000000000000000000000000 --- a/0002-LU-15821-ldlm-Prioritize-blocking-callbacks.patch +++ /dev/null @@ -1,159 +0,0 @@ -From 8ca1186151faa778edd5abd361e92fcd5d8ff56b Mon Sep 17 00:00:00 2001 -From: Patrick Farrell -Date: Wed, 4 May 2022 20:50:57 -0400 -Subject: [PATCH 02/61] LU-15821 ldlm: Prioritize blocking callbacks - -The current code places bl_ast lock callbacks at the end of -the global BL callback queue. This is bad because it -causes urgent requests from the server to wait behind -non-urgent cleanup tasks to keep lru_size at the right -level. - -This can lead to evictions if there is a large queue of -items in the global queue so the callback is not serviced -in a timely manner. - -Put bl_ast callbacks on the priority queue so they do not -wait behind the background traffic. - -Add some additional debug in this area. - -Lustre-change: https://review.whamcloud.com/47215 -Lustre-commit: 2d59294d52b696125acc464e5910c893d9aef237 - -Signed-off-by: Patrick Farrell -Change-Id: Ic6eb65819a4a93e9d30e807d386ca18380b30c7d -Reviewed-by: Andreas Dilger -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49610 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Stephane Thiell -Reviewed-by: Oleg Drokin ---- - lustre/ldlm/ldlm_lockd.c | 60 ++++++++++++++++++++++++++++++++-------- - 1 file changed, 48 insertions(+), 12 deletions(-) - -diff --git a/lustre/ldlm/ldlm_lockd.c b/lustre/ldlm/ldlm_lockd.c -index f82df7df0e..7e59709ea6 100644 ---- a/lustre/ldlm/ldlm_lockd.c -+++ b/lustre/ldlm/ldlm_lockd.c -@@ -83,27 +83,29 @@ static inline timeout_t ldlm_get_rq_timeout(void) - } - - struct ldlm_bl_pool { -- spinlock_t blp_lock; -+ spinlock_t blp_lock; - - /* - * blp_prio_list is used for callbacks that should be handled - * as a priority. It is used for LDLM_FL_DISCARD_DATA requests. - * see b=13843 - */ -- struct list_head blp_prio_list; -+ struct list_head blp_prio_list; - - /* - * blp_list is used for all other callbacks which are likely - * to take longer to process. - */ -- struct list_head blp_list; -- -- wait_queue_head_t blp_waitq; -- struct completion blp_comp; -- atomic_t blp_num_threads; -- atomic_t blp_busy_threads; -- int blp_min_threads; -- int blp_max_threads; -+ struct list_head blp_list; -+ -+ wait_queue_head_t blp_waitq; -+ struct completion blp_comp; -+ atomic_t blp_num_threads; -+ atomic_t blp_busy_threads; -+ int blp_min_threads; -+ int blp_max_threads; -+ int blp_total_locks; -+ int blp_total_blwis; - }; - - struct ldlm_bl_work_item { -@@ -2116,22 +2118,41 @@ static int __ldlm_bl_to_thread(struct ldlm_bl_work_item *blwi, - enum ldlm_cancel_flags cancel_flags) - { - struct ldlm_bl_pool *blp = ldlm_state->ldlm_bl_pool; -+ char *prio = "regular"; -+ int count; - - ENTRY; - - spin_lock(&blp->blp_lock); -+ /* cannot access blwi after added to list and lock is dropped */ -+ count = blwi->blwi_lock ? 1 : blwi->blwi_count; -+ -+ /* if the server is waiting on a lock to be cancelled (bl_ast), this is -+ * an urgent request and should go in the priority queue so it doesn't -+ * get stuck behind non-priority work (eg, lru size management) -+ * -+ * We also prioritize discard_data, which is for eviction handling -+ */ - if (blwi->blwi_lock && -- ldlm_is_discard_data(blwi->blwi_lock)) { -- /* add LDLM_FL_DISCARD_DATA requests to the priority list */ -+ (ldlm_is_discard_data(blwi->blwi_lock) || -+ ldlm_is_bl_ast(blwi->blwi_lock))) { - list_add_tail(&blwi->blwi_entry, &blp->blp_prio_list); -+ prio = "priority"; - } else { - /* other blocking callbacks are added to the regular list */ - list_add_tail(&blwi->blwi_entry, &blp->blp_list); - } -+ blp->blp_total_locks += count; -+ blp->blp_total_blwis++; - spin_unlock(&blp->blp_lock); - - wake_up(&blp->blp_waitq); - -+ /* unlocked read of blp values is intentional - OK for debug */ -+ CDEBUG(D_DLMTRACE, -+ "added %d/%d locks to %s blp list, %d blwis in pool\n", -+ count, blp->blp_total_locks, prio, blp->blp_total_blwis); -+ - /* - * can not check blwi->blwi_flags as blwi could be already freed in - * LCF_ASYNC mode -@@ -2749,10 +2770,23 @@ static int ldlm_bl_get_work(struct ldlm_bl_pool *blp, - if (++num_bl >= num_th) - num_bl = 0; - list_del(&blwi->blwi_entry); -+ blp->blp_total_locks -= blwi->blwi_lock ? 1 : blwi->blwi_count; -+ blp->blp_total_blwis--; - } - spin_unlock(&blp->blp_lock); - *p_blwi = blwi; - -+ /* intentional unlocked read of blp values - OK for debug */ -+ if (blwi) { -+ CDEBUG(D_DLMTRACE, -+ "Got %d locks of %d total in blp. (%d blwis in pool)\n", -+ blwi->blwi_lock ? 1 : blwi->blwi_count, -+ blp->blp_total_locks, blp->blp_total_blwis); -+ } else { -+ CDEBUG(D_DLMTRACE, -+ "No blwi found in queue (no bl locks in queue)\n"); -+ } -+ - if (*p_exp != NULL && *p_blwi != NULL) { - obd_stale_export_put(*p_exp); - *p_exp = NULL; -@@ -3293,6 +3327,8 @@ static int ldlm_setup(void) - init_waitqueue_head(&blp->blp_waitq); - atomic_set(&blp->blp_num_threads, 0); - atomic_set(&blp->blp_busy_threads, 0); -+ blp->blp_total_locks = 0; -+ blp->blp_total_blwis = 0; - - if (ldlm_num_threads == 0) { - blp->blp_min_threads = LDLM_NTHRS_INIT; --- -2.33.0 - diff --git a/0002-LU-16894-lod-fix-stripe_count-limit-in-lod_qos_set_p.patch b/0002-LU-16894-lod-fix-stripe_count-limit-in-lod_qos_set_p.patch new file mode 100644 index 0000000000000000000000000000000000000000..1923e667d877155b42595d75f1054b8444ff4e8d --- /dev/null +++ b/0002-LU-16894-lod-fix-stripe_count-limit-in-lod_qos_set_p.patch @@ -0,0 +1,124 @@ +From 6d679873f084aaf909ece93254c97ec4cea550e3 Mon Sep 17 00:00:00 2001 +From: Etienne AUJAMES +Date: Wed, 14 Jun 2023 10:02:01 +0200 +Subject: [PATCH 02/49] LU-16894 lod: fix stripe_count limit in + lod_qos_set_pool + +For a conflicting pool name and stripe offset parameter, the MDS +should not set the pool on the file layout in favor of the stripe +offset (LU-15658). But lod_qos_set_pool() still limit the +stripe_count to the number of OSTs in the specified pool. + +Let's fix this. + +Lustre-change: https://review.whamcloud.com/51314 +Lustre-commit: d6df160877cda3ff1e0b3ce5c4df46fa17e1468a + +Fixes: 06dd5a4 ("LU-15658 lod: ost list and pool name conflict") +Signed-off-by: Etienne AUJAMES +Change-Id: Ic47f9aadd8feea01367e526aaf0ea41a69ade9fa +Reviewed-by: Alexander Boyko +Reviewed-by: Andreas Dilger +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51527 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + lustre/lod/lod_qos.c | 70 +++++++++++++++++++++++--------------------- + 1 file changed, 37 insertions(+), 33 deletions(-) + +diff --git a/lustre/lod/lod_qos.c b/lustre/lod/lod_qos.c +index 0cb71d05c8..4d32877015 100644 +--- a/lustre/lod/lod_qos.c ++++ b/lustre/lod/lod_qos.c +@@ -2146,6 +2146,7 @@ static void lod_qos_set_pool(struct lod_object *lo, int pos, char *pool_name, + struct lod_device *d = lu2lod_dev(lod2lu_obj(lo)->lo_dev); + struct lod_layout_component *lod_comp; + struct pool_desc *pool = NULL; ++ __u32 idx; + int j, rc = 0; + + /* In the function below, .hs_keycmp resolves to +@@ -2154,43 +2155,46 @@ static void lod_qos_set_pool(struct lod_object *lo, int pos, char *pool_name, + if (pool_name) + pool = lod_find_pool(d, pool_name); + +- if (pool) { +- lod_comp = &lo->ldo_comp_entries[pos]; +- if (lod_comp->llc_stripe_offset != LOV_OFFSET_DEFAULT) { +- if (v1->lmm_magic == LOV_USER_MAGIC_SPECIFIC) { +- struct lov_user_md_v3 *v3; ++ if (!pool) ++ goto out_setpool; + +- v3 = (struct lov_user_md_v3 *)v1; +- for (j = 0; j < v3->lmm_stripe_count; j++) { +- __u32 num; +- +- num = lod_comp->llc_ostlist.op_array[j]; +- rc = lod_check_index_in_pool(num, pool); +- if (rc) +- break; +- } +- } else { +- rc = lod_check_index_in_pool( +- lod_comp->llc_stripe_offset, pool); +- } +- if (rc < 0) { +- CDEBUG(D_LAYOUT, "%s: index %u is not in the " +- "pool %s, dropping the pool\n", +- lod2obd(d)->obd_name, +- lod_comp->llc_stripe_offset, +- pool_name); +- pool_name = NULL; +- } +- } +- +- if (lod_comp->llc_stripe_count > pool_tgt_count(pool) && +- !(lod_comp->llc_pattern & LOV_PATTERN_OVERSTRIPING)) +- lod_comp->llc_stripe_count = pool_tgt_count(pool); ++ lod_comp = &lo->ldo_comp_entries[pos]; ++ if (lod_comp->llc_stripe_offset == LOV_OFFSET_DEFAULT) ++ goto out_checkcount; + +- lod_pool_putref(pool); +- } ++ if (v1->lmm_magic == LOV_USER_MAGIC_SPECIFIC) { ++ struct lov_user_md_v3 *v3; + ++ v3 = (struct lov_user_md_v3 *)v1; ++ for (j = 0; j < v3->lmm_stripe_count; j++) { ++ idx = lod_comp->llc_ostlist.op_array[j]; ++ rc = lod_check_index_in_pool(idx, pool); ++ if (rc) ++ break; ++ } ++ } else { ++ idx = lod_comp->llc_stripe_offset; ++ rc = lod_check_index_in_pool(idx, pool); ++ } ++ ++ if (!rc) ++ goto out_checkcount; ++ ++ CDEBUG(D_LAYOUT, ++ "%s: index %u is not in the pool %s, dropping the pool\n", ++ lod2obd(d)->obd_name, idx, pool_name); ++ pool_name = NULL; ++ goto out_putref; ++ ++out_checkcount: ++ if (lod_comp->llc_stripe_count > pool_tgt_count(pool) && ++ !(lod_comp->llc_pattern & LOV_PATTERN_OVERSTRIPING)) ++ lod_comp->llc_stripe_count = pool_tgt_count(pool); ++out_putref: ++ lod_pool_putref(pool); ++out_setpool: + lod_obj_set_pool(lo, pos, pool_name); ++ + } + + /** +-- +2.33.0 + diff --git a/0003-LU-14377-tests-make-parallel-scale-rr_alloc-less-str.patch b/0003-LU-14377-tests-make-parallel-scale-rr_alloc-less-str.patch deleted file mode 100644 index ba98d63af6b9b6c96f8ee253c0338438f4c6f511..0000000000000000000000000000000000000000 --- a/0003-LU-14377-tests-make-parallel-scale-rr_alloc-less-str.patch +++ /dev/null @@ -1,144 +0,0 @@ -From c0b60c0c79a2d5d5be651570564d6d0407457a5f Mon Sep 17 00:00:00 2001 -From: Andreas Dilger -Date: Tue, 18 Oct 2022 18:37:58 -0600 -Subject: [PATCH 03/61] LU-14377 tests: make parallel-scale/rr_alloc less - strict - -test_rr_alloc() sometimes fails with a difference of 3-4 objects -per OST, after creating 1500+ objects on each OST. This should -not be considered fatal. Make the test more lenient, and allow -a difference of up to 0.3% of objects between the OSTs. - -Fix some code style issues in the test. - -Lustre-change: https://review.whamcloud.com/48914 -Lustre-commit: b104c0a27713899a4d047f56fed57c30c39b8195 - -Test-Parameters: trivial testlist=parallel-scale env=ONLY=rr_alloc -Signed-off-by: Andreas Dilger -Change-Id: Ib6ba8c5d8e9d3245833448a52f8ed25308698a33 -Reviewed-by: Lai Siyao -Reviewed-by: Elena Gryaznova -(cherry picked from commit b104c0a27713899a4d047f56fed57c30c39b8195) -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51142 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/tests/functions.sh | 52 +++++++++++++++++++++------------------ - 1 file changed, 28 insertions(+), 24 deletions(-) - -diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh -index 1807bb8d17..3017fdb887 100644 ---- a/lustre/tests/functions.sh -+++ b/lustre/tests/functions.sh -@@ -1010,16 +1010,15 @@ cleanup_rr_alloc () { - - run_rr_alloc() { - remote_mds_nodsh && skip "remote MDS with nodsh" -- echo "===Test gives more reproduction percentage if number of "\ -- "client and ost are more. Test with 44 or more clients "\ -- "and 73 or more OSTs gives 100% reproduction rate==" -+ -+ echo "===Test gives more reproduction percentage if number of " -+ echo " client and ost are more. Test with 44 or more clients " -+ echo " and 73 or more OSTs gives 100% reproduction rate==" - - RR_ALLOC=${RR_ALLOC:-$(which rr_alloc 2> /dev/null || true)} - [ x$RR_ALLOC = x ] && skip_env "rr_alloc not found" - declare -a diff_max_min_arr -- # foeo = file on each ost. calc = calculated. - local ost_idx -- local foeo_calc - local qos_prec_objs="${TMP}/qos_and_precreated_objects" - local rr_alloc_NFILES=${rr_alloc_NFILES:-555} - local rr_alloc_MNTPTS=${rr_alloc_MNTPTS:-11} -@@ -1036,9 +1035,6 @@ run_rr_alloc() { - error_exit "Failed to mount lustre on ${mntpt_root}$i $clients" - done - -- local cmd="$RR_ALLOC $mntpt_root/$tdir/ash $rr_alloc_NFILES \ -- $num_clients" -- - # Save mdt values, set threshold to 100% i.e always Round Robin, - # restore the saved values again after creating files... - save_lustre_params mds1 \ -@@ -1056,7 +1052,8 @@ run_rr_alloc() { - # per OST are not multiple of that then it will be set to nearest - # lower power of 2. So set 'create_count' to the upper power of 2. - -- foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT)) -+ # foeo = file on each ost. calc = calculated. -+ local foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT)) - local create_count=$((2 * foeo_calc)) - - # create_count accepted values: -@@ -1078,18 +1075,19 @@ run_rr_alloc() { - # is created per OSTs. - createmany -o $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) \ - > /dev/null -- rm -f /$DIR/$tdir/foo* -+ unlinkmany $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) - - # Check for enough precreated objects... We should not - # fail here because code(osp_precreate.c) also takes care of it. - # So we have good chances of passing test even if this check fails. - local mdt_idx=0 -- for ost_idx in $(seq 0 $((OSTCOUNT - 1))); do -- [[ $(precreated_ost_obj_count $mdt_idx $ost_idx) -ge \ -- $foeo_calc ]] || echo "Warning: test may fail because" \ -- "of lack of precreated objects on OST${ost_idx}" -+ for ((ost_idx = 0; ost_idx < $OSTCOUNT; ost_idx++ )); do -+ (($(precreated_ost_obj_count $mdt_idx $ost_idx) >= foeo_calc))|| -+ echo "Warning: test may fail from too few objs on OST$ost_idx" - done - -+ local cmd="$RR_ALLOC $mntpt_root/$tdir/f $rr_alloc_NFILES $num_clients" -+ - if [[ $total_MNTPTS -ne 0 ]]; then - # Now start the actual file creation app. - mpi_run "-np $total_MNTPTS" $cmd || return -@@ -1101,12 +1099,14 @@ run_rr_alloc() { - rm -f $qos_prec_objs - - diff_max_min_arr=($($LFS getstripe -r $DIR/$tdir/ | -- grep "lmm_stripe_offset:" | awk '{print $2}' | sort -n | -- uniq -c | awk 'NR==1 {min=max=$1} \ -- { $1max ? max=$1 : max} \ -- END {print max-min, max, min}')) -+ awk '/lmm_stripe_offset:/ {print $2}' | -+ sort | uniq -c | -+ awk 'NR==1 {min=max=$1} \ -+ { $1max ? max=$1:max} \ -+ END {print max-min, max, min}')) -+ -+ $LFS find $DIR/$tdir -type f | xargs -n1 -P8 unlink - -- rm -rf $DIR/$tdir - - # In-case of fairly large number of file creation using RR (round-robin) - # there can be two cases in which deviation will occur than the regular -@@ -1114,11 +1114,15 @@ run_rr_alloc() { - # 1- When rr_alloc does not start right with 'lqr_start_count' reseeded, - # 2- When rr_alloc does not finish with 'lqr_start_count == 0'. - # So the difference of files b/w any 2 OST should not be more than 2. -- [[ ${diff_max_min_arr[0]} -le 2 ]] || -- error "Uneven distribution detected: difference between" \ -- "maximum files per OST (${diff_max_min_arr[1]}) and" \ -- "minimum files per OST (${diff_max_min_arr[2]}) must not be" \ -- "greater than 2" -+ # In some cases it may be more, but shouldn't be > 0.3% of the files. -+ local max_diff=$((create_count > 600 ? create_count / 300 : 2)) -+ -+ (( ${diff_max_min_arr[0]} <= $max_diff )) || { -+ $LFS getstripe -r $DIR/$tdir | -+ awk '/lmm_stripe_offset:/ {print $2}' | sort | uniq -c -+ -+ error "max/min OST objects (${diff_max_min_arr[1]} : ${diff_max_min_arr[2]}) too different" -+ } - } - - run_fs_test() { --- -2.33.0 - diff --git a/0003-LU-15595-tests-Add-various-router-tests.patch b/0003-LU-15595-tests-Add-various-router-tests.patch new file mode 100644 index 0000000000000000000000000000000000000000..0ef154cac980f50cf449a89ca520daa273c4191a --- /dev/null +++ b/0003-LU-15595-tests-Add-various-router-tests.patch @@ -0,0 +1,449 @@ +From 19dbb06af087e9a754bf803c40a0ae12139c6d43 Mon Sep 17 00:00:00 2001 +From: Chris Horn +Date: Mon, 7 Feb 2022 23:20:37 +0000 +Subject: [PATCH 03/49] LU-15595 tests: Add various router tests + +Add test cases to exercise LNet routing. + +Lustre-change: https://review.whamcloud.com/46622 +Lustre-commit: 8ee85e15412d32fbe60f70c474c0a28ff15b8351 + +Test-Parameters: trivial testlist=sanity-lnet +Signed-off-by: Chris Horn +Change-Id: I4a077937b3e3b8b07707afeb0c5c23ec1c9074f4 +Reviewed-by: Cyril Bordage +Reviewed-by: James Simmons +Signed-off-by: Etienne AUJAMES +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51543 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + lustre/tests/sanity-lnet.sh | 411 ++++++++++++++++++++++++++++++++++++ + 1 file changed, 411 insertions(+) + +diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh +index 1fb298776c..88f2b815cb 100755 +--- a/lustre/tests/sanity-lnet.sh ++++ b/lustre/tests/sanity-lnet.sh +@@ -2435,6 +2435,417 @@ test_219() { + } + run_test 219 "Consolidate peer entries" + ++do_net_add() { ++ local node=$1 ++ local net=$2 ++ local if=$3 ++ ++ do_rpc_nodes $node "$LNETCTL net add --net $net --if $if" || ++ error "add $net on interface $if on node $node failed rc=$?" ++} ++ ++do_route_add() { ++ local node=$1 ++ local net=$2 ++ local gw=$3 ++ ++ do_node $node "$LNETCTL route add --net $net --gateway $gw" || ++ error "route add to $net via $gw failed rc=$?" ++} ++ ++ROUTER="" ++ROUTER_INTERFACES=() ++RPEER="" ++RPEER_INTERFACES=() ++init_router_test_vars() { ++ local rnodes=$(remote_nodes_list) ++ [[ -z $rnodes || $(wc -w <<<$rnodes) -lt 2 ]] && ++ skip "Need at least 2 remote nodes found \"$rnodes\"" ++ ++ ROUTER=$(awk '{print $1}' <<<$rnodes) ++ RPEER=$(awk '{print $2}' <<<$rnodes) ++ ++ rnodes=$(comma_list $ROUTER $RPEER) ++ local all_nodes=$(comma_list $rnodes $HOSTNAME) ++ ++ do_nodes $rnodes $LUSTRE_RMMOD || ++ error "failed to unload modules" ++ ++ do_rpc_nodes $rnodes "load_lnet config_on_load=1" || ++ error "Failed to load and configure LNet" ++ ++ ROUTER_INTERFACES=( $(do_rpc_nodes --quiet $ROUTER lnet_if_list) ) ++ ++ RPEER_INTERFACES=( $(do_rpc_nodes --quiet $RPEER lnet_if_list) ) ++ ++ do_nodes $all_nodes $LUSTRE_RMMOD || ++ error "Failed to unload modules" ++ ++ [[ ${#INTERFACES[@]} -eq 0 ]] && ++ error "No interfaces configured for local host $HOSTNAME" ++ [[ ${#ROUTER_INTERFACES[@]} -eq 0 ]] && ++ error "No interfaces configured for router $ROUTER" ++ [[ ${#RPEER_INTERFACES[@]} -eq 0 ]] && ++ error "No interfaces configured for remote peer $RPEER" ++ ++ return 0 ++} ++ ++ROUTER_NIDS=() ++RPEER_NIDS=() ++LNIDS=() ++LOCAL_NET=${NETTYPE}1 ++REMOTE_NET=${NETTYPE}2 ++setup_router_test() { ++ local mod_opts="$@" ++ ++ if [[ ${#RPEER_INTERFACES[@]} -eq 0 ]]; then ++ init_router_test_vars || ++ return $? ++ fi ++ ++ local all_nodes=$(comma_list $ROUTER $RPEER $HOSTNAME) ++ ++ do_nodes $all_nodes $LUSTRE_RMMOD || ++ error "failed to unload modules" ++ ++ mod_opts+=" alive_router_check_interval=5" ++ mod_opts+=" router_ping_timeout=5" ++ do_rpc_nodes $all_nodes load_lnet "${mod_opts}" || ++ error "Failed to load lnet" ++ ++ do_nodes $all_nodes "$LNETCTL lnet configure" || ++ error "Failed to initialize DLC" ++ ++ do_net_add $ROUTER $LOCAL_NET ${ROUTER_INTERFACES[0]} || ++ return $? ++ ++ do_net_add $ROUTER $REMOTE_NET ${ROUTER_INTERFACES[0]} || ++ return $? ++ ++ do_net_add $RPEER $REMOTE_NET ${RPEER_INTERFACES[0]} || ++ return $? ++ ++ add_net $LOCAL_NET ${INTERFACES[0]} || ++ return $? ++ ++ ROUTER_NIDS=( $(do_node $ROUTER $LCTL list_nids 2>/dev/null | ++ xargs echo) ) ++ RPEER_NIDS=( $(do_node $RPEER $LCTL list_nids 2>/dev/null | ++ xargs echo) ) ++ LNIDS=( $($LCTL list_nids 2>/dev/null | xargs echo) ) ++} ++ ++do_route_del() { ++ local node=$1 ++ local net=$2 ++ local gw=$3 ++ ++ do_nodesv $node "if $LNETCTL route show --net $net --gateway $gw; then \ ++ $LNETCTL route del --net $net --gateway $gw; \ ++ else \ ++ exit 0; \ ++ fi" ++} ++ ++cleanup_router_test() { ++ local all_nodes=$(comma_list $HOSTNAME $ROUTER $RPEER) ++ ++ do_route_del $HOSTNAME $REMOTE_NET ${ROUTER_NIDS[0]} || ++ error "Failed to delete $REMOTE_NET route" ++ ++ do_route_del $RPEER $LOCAL_NET ${ROUTER_NIDS[1]} || ++ error "Failed to delete $LOCAL_NET route" ++ ++ do_nodes $all_nodes $LUSTRE_RMMOD || ++ error "failed to unload modules" ++ ++ return 0 ++} ++ ++check_route_aliveness() { ++ local node="$1" ++ local expected="$2" ++ ++ local lctl_actual ++ local lnetctl_actual ++ local chk_intvl ++ local i ++ ++ chk_intvl=$(cat /sys/module/lnet/parameters/alive_router_check_interval) ++ ++ lctl_actual=$(do_node $node $LCTL show_route | awk '{print $7}') ++ lnetctl_actual=$(do_node $node $LNETCTL route show -v | ++ awk '/state/{print $NF}') ++ ++ for ((i = 0; i < $chk_intvl; i++)); do ++ if [[ $lctl_actual == $expected ]] && ++ [[ $lnetctl_actual == $expected ]]; then ++ break ++ fi ++ ++ echo "wait 1s for route state change" ++ sleep 1 ++ ++ lctl_actual=$(do_node $node $LCTL show_route | awk '{print $7}') ++ lnetctl_actual=$(do_node $node $LNETCTL route show -v | ++ awk '/state/{print $NF}') ++ done ++ ++ [[ $lctl_actual != $expected ]] && ++ error "Wanted \"$expected\" lctl found \"$lctl_actual\"" ++ ++ [[ $lnetctl_actual != $expected ]] && ++ error "Wanted \"$expected\" lnetctl found \"$lnetctl_actual\"" ++ ++ return 0 ++} ++ ++check_router_ni_status() { ++ local expected_local="$1" ++ local expected_remote="$2" ++ ++ local actual_local ++ local actual_remote ++ local chk_intvl ++ local timeout ++ local i ++ ++ chk_intvl=$(cat /sys/module/lnet/parameters/alive_router_check_interval) ++ timeout=$(cat /sys/module/lnet/parameters/router_ping_timeout) ++ ++ actual_local=$(do_node $ROUTER "$LNETCTL net show --net $LOCAL_NET" | ++ awk '/status/{print $NF}') ++ actual_remote=$(do_node $ROUTER "$LNETCTL net show --net $REMOTE_NET" | ++ awk '/status/{print $NF}') ++ ++ for ((i = 0; i < $((chk_intvl + timeout)); i++)); do ++ if [[ $actual_local == $expected_local ]] && ++ [[ $actual_remote == $expected_remote ]]; then ++ break ++ fi ++ ++ echo "wait 1s for NI state change" ++ sleep 1 ++ ++ actual_local=$(do_node $ROUTER \ ++ "$LNETCTL net show --net $LOCAL_NET" | ++ awk '/status/{print $NF}') ++ actual_remote=$(do_node $ROUTER \ ++ "$LNETCTL net show --net $REMOTE_NET" | ++ awk '/status/{print $NF}') ++ done ++ ++ [[ $actual_local == $expected_local ]] || ++ error "$LOCAL_NET should be $expected_local" ++ ++ [[ $actual_remote == $expected_remote ]] || ++ error "$REMOTE_NET should be $expected_remote" ++ ++ return 0 ++} ++ ++do_basic_rtr_test() { ++ do_node $ROUTER "$LNETCTL set routing 1" || ++ error "Unable to enable routing on $ROUTER" ++ ++ do_route_add $HOSTNAME $REMOTE_NET ${ROUTER_NIDS[0]} || ++ return $? ++ ++ do_route_add $RPEER $LOCAL_NET ${ROUTER_NIDS[1]} || ++ return $? ++ ++ check_route_aliveness "$HOSTNAME" "up" || ++ return $? ++ ++ check_route_aliveness "$RPEER" "up" || ++ return $? ++ ++ do_lnetctl ping ${RPEER_NIDS[0]} || ++ error "Failed to ping ${RPEER_NIDS[0]}" ++ ++ do_node $RPEER "$LNETCTL ping ${LNIDS[0]}" || ++ error "$RPEER failed to ping ${LNIDS[0]}" ++ ++ return 0 ++} ++ ++test_220() { ++ setup_router_test || return $? ++ ++ do_basic_rtr_test || return $? ++ ++ cleanup_router_test || return $? ++} ++run_test 220 "Add routes w/default options - check aliveness" ++ ++test_221() { ++ setup_router_test lnet_peer_discovery_disabled=1 || return $? ++ ++ do_basic_rtr_test || return $? ++ ++ cleanup_router_test || return $? ++} ++run_test 221 "Add routes w/DD disabled - check aliveness" ++ ++do_aarf_enabled_test() { ++ do_node $ROUTER "$LNETCTL set routing 1" || ++ error "Unable to enable routing on $ROUTER" ++ ++ check_router_ni_status "down" "down" ++ ++ do_lnetctl ping ${RPEER_NIDS[0]} && ++ error "Ping should fail" ++ ++ do_node $RPEER "$LNETCTL ping ${LNIDS[0]}" && ++ error "$RPEER ping should fail" ++ ++ # Adding a route should cause the router's NI on LOCAL_NET to get up ++ do_route_add $HOSTNAME $REMOTE_NET ${ROUTER_NIDS[0]} || ++ return $? ++ ++ check_router_ni_status "up" "down" || ++ return $? ++ ++ # But route should still be down because of avoid_asym_router_failure ++ check_route_aliveness "$HOSTNAME" "down" || ++ return $? ++ ++ do_lnetctl ping ${RPEER_NIDS[0]} && ++ error "Ping should fail" ++ ++ do_node $RPEER "$LNETCTL ping ${LNIDS[0]}" && ++ error "$RPEER ping should fail" ++ ++ # Adding the symmetric route should cause the remote NI to go up and ++ # routes to go up ++ do_route_add $RPEER $LOCAL_NET ${ROUTER_NIDS[1]} || ++ return $? ++ ++ check_router_ni_status "up" "up" || ++ return $? ++ ++ check_route_aliveness "$HOSTNAME" "up" || ++ return $? ++ ++ check_route_aliveness "$RPEER" "up" || ++ return $? ++ ++ do_lnetctl ping ${RPEER_NIDS[0]} || ++ error "Failed to ping ${RPEER_NIDS[0]}" ++ ++ do_node $RPEER "$LNETCTL ping ${LNIDS[0]}" || ++ error "$RPEER failed to ping ${LNIDS[0]}" ++ ++ # Stop LNet on local host ++ do_lnetctl lnet unconfigure || ++ error "Failed to stop LNet rc=$?" ++ ++ check_router_ni_status "down" "up" || ++ return $? ++ ++ check_route_aliveness "$RPEER" "down" || ++ return $? ++ ++ do_lnetctl ping ${RPEER_NIDS[0]} && ++ error "Ping should fail" ++ ++ do_node $RPEER "$LNETCTL ping ${LNIDS[0]}" && ++ error "$RPEER ping should fail" ++ ++ return 0 ++} ++ ++test_222() { ++ setup_router_test avoid_asym_router_failure=1 || return $? ++ ++ do_aarf_enabled_test || return $? ++ ++ cleanup_router_test || return $? ++} ++run_test 222 "Check avoid_asym_router_failure=1" ++ ++test_223() { ++ local opts="avoid_asym_router_failure=1 lnet_peer_discovery_disabled=1" ++ ++ setup_router_test $opts || return $? ++ ++ do_aarf_enabled_test || return $? ++ ++ cleanup_router_test || return $? ++} ++run_test 223 "Check avoid_asym_router_failure=1 w/DD disabled" ++ ++do_aarf_disabled_test() { ++ do_node $ROUTER "$LNETCTL set routing 1" || ++ error "Unable to enable routing on $ROUTER" ++ ++ check_router_ni_status "down" "down" ++ ++ do_route_add $HOSTNAME $REMOTE_NET ${ROUTER_NIDS[0]} || ++ return $? ++ ++ check_router_ni_status "up" "down" || ++ return $? ++ ++ check_route_aliveness "$HOSTNAME" "up" || ++ return $? ++ ++ do_route_add $RPEER $LOCAL_NET ${ROUTER_NIDS[1]} || ++ return $? ++ ++ check_router_ni_status "up" "up" || ++ return $? ++ ++ check_route_aliveness "$HOSTNAME" "up" || ++ return $? ++ ++ check_route_aliveness "$RPEER" "up" || ++ return $? ++ ++ do_lnetctl ping ${RPEER_NIDS[0]} || ++ error "Failed to ping ${RPEER_NIDS[0]}" ++ ++ do_node $RPEER "$LNETCTL ping ${LNIDS[0]}" || ++ error "$RPEER failed to ping ${LNIDS[0]}" ++ ++ # Stop LNet on local host ++ do_lnetctl lnet unconfigure || ++ error "Failed to stop LNet rc=$?" ++ ++ check_router_ni_status "down" "up" || ++ return $? ++ ++ check_route_aliveness "$RPEER" "up" || ++ return $? ++ ++ return 0 ++} ++ ++test_224() { ++ setup_router_test avoid_asym_router_failure=0 || ++ return $? ++ ++ do_aarf_disabled_test || ++ return $? ++ ++ cleanup_router_test || ++ return $? ++} ++run_test 224 "Check avoid_asym_router_failure=0" ++ ++test_225() { ++ local opts="avoid_asym_router_failure=0 lnet_peer_discovery_disabled=1" ++ ++ setup_router_test $opts || return $? ++ ++ do_aarf_disabled_test || return $? ++ ++ cleanup_router_test || ++ return $? ++} ++run_test 225 "Check avoid_asym_router_failure=0 w/DD disabled" ++ + test_230() { + # LU-12815 + echo "Check valid values; Should succeed" +-- +2.33.0 + diff --git a/0058-LU-14073-ldiskfs-don-t-test-LDISKFS_IOC_FSSETXATTR.patch b/0004-LU-14073-ldiskfs-don-t-test-LDISKFS_IOC_FSSETXATTR.patch similarity index 83% rename from 0058-LU-14073-ldiskfs-don-t-test-LDISKFS_IOC_FSSETXATTR.patch rename to 0004-LU-14073-ldiskfs-don-t-test-LDISKFS_IOC_FSSETXATTR.patch index 9702f9bf13d762f7305a0967b47f977e515690d9..fc6d6a735429e90db4eb18765a898b8ed5846639 100644 --- a/0058-LU-14073-ldiskfs-don-t-test-LDISKFS_IOC_FSSETXATTR.patch +++ b/0004-LU-14073-ldiskfs-don-t-test-LDISKFS_IOC_FSSETXATTR.patch @@ -1,7 +1,7 @@ -From 827664003a452e249ed423cb2113dbbbae28e3a9 Mon Sep 17 00:00:00 2001 +From 9eb23bdd3833911d02bf7839506a5d5459299040 Mon Sep 17 00:00:00 2001 From: Mr NeilBrown Date: Fri, 9 Dec 2022 16:31:13 +1100 -Subject: [PATCH 58/61] LU-14073 ldiskfs: don't test LDISKFS_IOC_FSSETXATTR +Subject: [PATCH 04/49] LU-14073 ldiskfs: don't test LDISKFS_IOC_FSSETXATTR EXT4_IOC_FSSETXATTR was removed upstream in Linux 5.9, Commit cb29a02d3a9d ("ext4: use generic names for generic ioctls"). @@ -11,24 +11,25 @@ Instead test if EXT4_MAXQUOTAS is 3. This was changed to 3 upstream in the commit immediately before EXT4_IOC_FSSETXATTR was added, so it is effectively the same test. +Lustre-change: https://review.whamcloud.com/49353 +Lustre-commit: 40389067f5645fed903304d19dd6c58de72d0b88 + Test-Parameters: trivial Signed-off-by: Mr NeilBrown Change-Id: I88c51c03959ebe98cd5066596f5158fac570a625 -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49353 -Tested-by: jenkins -Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Andreas Dilger -Reviewed-by: Oleg Drokin -Signed-off-by: Xinliang Liu -(cherry picked from commit 40389067f5645fed903304d19dd6c58de72d0b88) Signed-off-by: Xinliang Liu +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/52283 +Reviewed-by: Oleg Drokin +Tested-by: jenkins +Tested-by: Maloo --- lustre/osd-ldiskfs/osd_internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h -index 7300278b80..baf2dc9baa 100644 +index f06dde1b00..96de28d311 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -101,7 +101,7 @@ extern struct kmem_cache *dynlock_cachep; diff --git a/0004-LU-15123-tests-check-quota-reintegration-after-recov.patch b/0004-LU-15123-tests-check-quota-reintegration-after-recov.patch deleted file mode 100644 index 003037814ffb440f9228c1b407d0398c23c04029..0000000000000000000000000000000000000000 --- a/0004-LU-15123-tests-check-quota-reintegration-after-recov.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 13805e3a2d4f520e297bc408d94b9971a6094f9a Mon Sep 17 00:00:00 2001 -From: Alex Zhuravlev -Date: Wed, 19 Apr 2023 10:20:33 +0300 -Subject: [PATCH 04/61] LU-15123 tests: check quota reintegration after - recovery - -4th step of quota reintegration (reconciliation) waits for recovery -completion. So the tests (like sanity-quota/7a) should wait for -recovery completion before checking reintegration results. - -Lustre-change: https://review.whamcloud.com/50688 -Lustre-commit: 4432b6e2824775e292f96e202d6fc0db231bc749 - -Signed-off-by: Alex Zhuravlev -Change-Id: Id0aa5db01658621103d94ad6dafe91b2960b3a33 -Reviewed-by: Andreas Dilger -Reviewed-by: Sergey Cheremencev -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51233 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/tests/sanity-quota.sh | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh -index f552d84fd6..047b1a9687 100755 ---- a/lustre/tests/sanity-quota.sh -+++ b/lustre/tests/sanity-quota.sh -@@ -285,6 +285,8 @@ wait_reintegration() { - cmd=${cmd}.quota_slave.info - - if $(facet_up $tgt); then -+ # reintegration starts after recovery completion -+ wait_recovery_complete $tgt - wait_update_facet $tgt "$cmd | - grep "$qtype" | awk '{ print \\\$3 }'" \ - "$result" $max || return 1 --- -2.33.0 - diff --git a/0005-LU-13081-tests-skip-sanity-test_151-test_156.patch b/0005-LU-13081-tests-skip-sanity-test_151-test_156.patch deleted file mode 100644 index f8c97cbddef4d906fb9e0286f2ab426a38d902bd..0000000000000000000000000000000000000000 --- a/0005-LU-13081-tests-skip-sanity-test_151-test_156.patch +++ /dev/null @@ -1,57 +0,0 @@ -From d15f9619eec367805606f4a46d049e6ecb62f36d Mon Sep 17 00:00:00 2001 -From: Alex Deiter -Date: Thu, 27 Apr 2023 02:04:01 +0400 -Subject: [PATCH 05/61] LU-13081 tests: skip sanity test_151/test_156 - -Skip both sanity test_151 and test_156 during interop testing, -since this is really testing server-side functionality only -(OSS caching behavior). And it makes sense to just exclude -test_151 and test_156 during interop testing, otherwise it -seems that the client version of the test can become -inconsistent with the caching behavior/tunables on the OSS -and the failures don't mean anything. There is enough -non-interop testing to catch any regressions in the OSS -cache behavior. - -Lustre-change: https://review.whamcloud.com/50777 -Lustre-commit: 305dda878d1dde822eab7a9dacfe8dec0b96cb3e - -Test-Parameters: trivial -Signed-off-by: Alex Deiter -Change-Id: I39a8b54894d5b0c7573e6c56d1f8e1ba02b3e3fe -Reviewed-by: Jian Yu -Reviewed-by: Andreas Dilger -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51286 -Reviewed-by: Alex Deiter -Reviewed-by: Oleg Drokin -Tested-by: jenkins -Tested-by: Maloo ---- - lustre/tests/sanity.sh | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh -index fec778e9fe..d73b285eca 100755 ---- a/lustre/tests/sanity.sh -+++ b/lustre/tests/sanity.sh -@@ -15079,6 +15079,8 @@ function set_cache() { - test_151() { - [ $PARALLEL == "yes" ] && skip "skip parallel run" - remote_ost_nodsh && skip "remote OST with nodsh" -+ (( CLIENT_VERSION == OST1_VERSION )) || -+ skip "LU-13081: no interop testing for OSS cache" - - local CPAGES=3 - local list=$(comma_list $(osts_nodes)) -@@ -15754,6 +15756,8 @@ test_156() { - skip "stats not implemented on old servers" - [ "$ost1_FSTYPE" = "zfs" ] && - skip "LU-1956/LU-2261: stats not implemented on OSD ZFS" -+ (( CLIENT_VERSION == OST1_VERSION )) || -+ skip "LU-13081: no interop testing for OSS cache" - - local CPAGES=3 - local BEFORE --- -2.33.0 - diff --git a/0005-LU-14824-test-sanity-413a-b-unlink-timeout-v2.patch b/0005-LU-14824-test-sanity-413a-b-unlink-timeout-v2.patch new file mode 100644 index 0000000000000000000000000000000000000000..43ae7e4d4a29e8e537b066c80fcc11855317d239 --- /dev/null +++ b/0005-LU-14824-test-sanity-413a-b-unlink-timeout-v2.patch @@ -0,0 +1,510 @@ +From 812542c9ef3ed85696b91f0c588b3836b2c329ce Mon Sep 17 00:00:00 2001 +From: Etienne AUJAMES +Date: Tue, 4 Jul 2023 17:16:31 +0200 +Subject: [PATCH 05/49] LU-14824 test: sanity 413a/b unlink timeout v2 + +Unlinking remote/striped directories is slow on ZFS system, limit +total directory number for 1-stripe directory test in 413a/b on ZFS +system, and don't test striped directory to avoid timeout. + +Also limit total stripe object count to avoid timeout. + +Use fallocate to fill the MDTs if it is supported. Add a new helper +function check_fallocate() that just determines if fallocate support +is available on the OST without trying to change the mode. This is +cached across calls to avoid repeated SSH calls to get information +that is the same for the entire test run. + +Lustre-change: https://review.whamcloud.com/49799 +Lustre-commit: 09fc9ccb1acc534b3fb074433dcbcecebb175384 + +The backport use parts of 37c1ddc ("LU-15850 lmv: always space-balance +r-r directories") to merge sanity.sh. + +Test-Parameters: trivial +Test-Parameters: mdscount=2 mdtcount=4 fstype=ldiskfs testlist=sanity env=ONLY="413a 413b",ONLY_REPEAT=50 +Test-Parameters: mdscount=2 mdtcount=4 fstype=zfs testlist=sanity env=ONLY="413a 413b",ONLY_REPEAT=50 +Signed-off-by: Lai Siyao +Change-Id: Ie116e6df5aee3877ed9f093f58e7bd71f63ebbe5 +Reviewed-by: Andreas Dilger +Reviewed-by: Arshad Hussain +Signed-off-by: Etienne AUJAMES +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51561 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + lustre/tests/sanity.sh | 189 ++++++++++++++++++++++----------- + lustre/tests/test-framework.sh | 55 ++++++++-- + 2 files changed, 176 insertions(+), 68 deletions(-) + +diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh +index 41110ac575..4f79e27e13 100755 +--- a/lustre/tests/sanity.sh ++++ b/lustre/tests/sanity.sh +@@ -25513,10 +25513,32 @@ test_412() { + } + run_test 412 "mkdir on specific MDTs" + +-generate_uneven_mdts() { +- local threshold=$1 ++TEST413_COUNT=${TEST413_COUNT:-200} ++ ++# ++# set_maxage() is used by test_413 only. ++# This is a helper function to set maxage. Does not return any value. ++# Input: maxage to set ++# ++set_maxage() { + local lmv_qos_maxage + local lod_qos_maxage ++ local new_maxage=$1 ++ ++ lmv_qos_maxage=$($LCTL get_param -n lmv.*.qos_maxage) ++ $LCTL set_param lmv.*.qos_maxage=$new_maxage ++ stack_trap "$LCTL set_param \ ++ lmv.*.qos_maxage=$lmv_qos_maxage > /dev/null" ++ lod_qos_maxage=$(do_facet mds1 $LCTL get_param -n \ ++ lod.$FSNAME-MDT0000-mdtlov.qos_maxage | awk '{ print $1 }') ++ do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \ ++ lod.*.mdt_qos_maxage=$new_maxage ++ stack_trap "do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \ ++ lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null" ++} ++ ++generate_uneven_mdts() { ++ local threshold=$1 + local ffree + local bavail + local max +@@ -25526,17 +25548,6 @@ generate_uneven_mdts() { + local tmp + local i + +- lmv_qos_maxage=$($LCTL get_param -n lmv.*.qos_maxage) +- $LCTL set_param lmv.*.qos_maxage=1 +- stack_trap "$LCTL set_param \ +- lmv.*.qos_maxage=$lmv_qos_maxage > /dev/null" RETURN +- lod_qos_maxage=$(do_facet mds1 $LCTL get_param -n \ +- lod.$FSNAME-MDT0000-mdtlov.qos_maxage | awk '{ print $1 }') +- do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \ +- lod.*.mdt_qos_maxage=1 +- stack_trap "do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \ +- lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null" RETURN +- + echo + echo "Check for uneven MDTs: " + +@@ -25560,36 +25571,49 @@ generate_uneven_mdts() { + fi + done + ++ (( min > 0 )) || skip "low space on MDT$min_index" + (( ${ffree[min_index]} > 0 )) || +- skip "no free files in MDT$min_index" ++ skip "no free files on MDT$min_index" + (( ${ffree[min_index]} < 10000000 )) || +- skip "too many free files in MDT$min_index" ++ skip "too many free files on MDT$min_index" + + # Check if we need to generate uneven MDTs + local diff=$(((max - min) * 100 / min)) +- local testdir=$DIR/$tdir-fillmdt ++ local testdirp=$DIR/$tdir-fillmdt # parent fill folder ++ local testdir # individual folder within $testdirp + local start ++ local cmd + +- mkdir -p $testdir ++ # fallocate is faster to consume space on MDT, if available ++ if check_fallocate_supported mds$((min_index + 1)); then ++ cmd="fallocate -l 128K " ++ else ++ cmd="dd if=/dev/zero bs=128K count=1 of=" ++ fi + +- i=0 +- while (( diff < threshold )); do ++ echo "using cmd $cmd" ++ for (( i = 0; diff < threshold; i++ )); do ++ testdir=${testdirp}/$i ++ [ -d $testdir ] && continue ++ ++ (( i % 10 > 0 )) || { $LFS df; $LFS df -i; } ++ ++ mkdir -p $testdirp + # generate uneven MDTs, create till $threshold% diff + echo -n "weight diff=$diff% must be > $threshold% ..." +- echo "Fill MDT$min_index with 1000 files: loop $i" +- testdir=$DIR/$tdir-fillmdt/$i +- [ -d $testdir ] || $LFS mkdir -i $min_index $testdir || ++ echo "Fill MDT$min_index with $TEST413_COUNT files: loop $i" ++ $LFS mkdir -i $min_index $testdir || + error "mkdir $testdir failed" + $LFS setstripe -E 1M -L mdt $testdir || + error "setstripe $testdir failed" + start=$SECONDS +- for F in f.{0..999}; do +- dd if=/dev/zero of=$testdir/$F bs=64K count=1 > \ +- /dev/null 2>&1 || error "dd $F failed" ++ for (( f = 0; f < TEST413_COUNT; f++ )); do ++ $cmd$testdir/f.$f &> /dev/null || error "$cmd $f failed" + done ++ sync; sleep 1; sync + + # wait for QOS to update +- (( SECONDS < start + 1 )) && sleep $((start + 1 - SECONDS)) ++ (( SECONDS < start + 2 )) && sleep $((start + 2 - SECONDS)) + + ffree=($(lctl get_param -n mdc.*[mM][dD][cC]-*.filesfree)) + bavail=($(lctl get_param -n mdc.*[mM][dD][cC]-*.kbytesavail)) +@@ -25597,8 +25621,8 @@ generate_uneven_mdts() { + (${bavail[max_index]} * bsize >> 16))) + min=$(((${ffree[min_index]} >> 8) * + (${bavail[min_index]} * bsize >> 16))) ++ (( min > 0 )) || skip "low space on MDT$min_index" + diff=$(((max - min) * 100 / min)) +- i=$((i + 1)) + done + + echo "MDT filesfree available: ${ffree[*]}" +@@ -25614,25 +25638,28 @@ test_qos_mkdir() { + local testdir + local lmv_qos_prio_free + local lmv_qos_threshold_rr +- local lmv_qos_maxage + local lod_qos_prio_free + local lod_qos_threshold_rr +- local lod_qos_maxage ++ local total + local count + local i + ++ # @total is total directories created if it's testing plain ++ # directories, otherwise it's total stripe object count for ++ # striped directories test. ++ # remote/striped directory unlinking is slow on zfs and may ++ # timeout, test with fewer directories ++ [ "$mds1_FSTYPE" = "zfs" ] && total=120 || total=240 ++ + lmv_qos_prio_free=$($LCTL get_param -n lmv.*.qos_prio_free | head -n1) + lmv_qos_prio_free=${lmv_qos_prio_free%%%} + lmv_qos_threshold_rr=$($LCTL get_param -n lmv.*.qos_threshold_rr | + head -n1) + lmv_qos_threshold_rr=${lmv_qos_threshold_rr%%%} +- lmv_qos_maxage=$($LCTL get_param -n lmv.*.qos_maxage) + stack_trap "$LCTL set_param \ + lmv.*.qos_prio_free=$lmv_qos_prio_free > /dev/null" + stack_trap "$LCTL set_param \ + lmv.*.qos_threshold_rr=$lmv_qos_threshold_rr > /dev/null" +- stack_trap "$LCTL set_param \ +- lmv.*.qos_maxage=$lmv_qos_maxage > /dev/null" + + lod_qos_prio_free=$(do_facet mds1 $LCTL get_param -n \ + lod.$FSNAME-MDT0000-mdtlov.mdt_qos_prio_free | head -n1) +@@ -25640,14 +25667,10 @@ test_qos_mkdir() { + lod_qos_threshold_rr=$(do_facet mds1 $LCTL get_param -n \ + lod.$FSNAME-MDT0000-mdtlov.mdt_qos_threshold_rr | head -n1) + lod_qos_threshold_rr=${lod_qos_threshold_rr%%%} +- lod_qos_maxage=$(do_facet mds1 $LCTL get_param -n \ +- lod.$FSNAME-MDT0000-mdtlov.qos_maxage | awk '{ print $1 }') + stack_trap "do_nodes $mdts $LCTL set_param \ + lod.*.mdt_qos_prio_free=$lod_qos_prio_free > /dev/null" + stack_trap "do_nodes $mdts $LCTL set_param \ + lod.*.mdt_qos_threshold_rr=$lod_qos_threshold_rr > /dev/null" +- stack_trap "do_nodes $mdts $LCTL set_param \ +- lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null" + + $LCTL set_param lmv.*.qos_threshold_rr=100 > /dev/null + do_nodes $mdts $LCTL set_param lod.*.mdt_qos_threshold_rr=100 > /dev/null +@@ -25669,8 +25692,8 @@ test_qos_mkdir() { + echo "Mkdir (stripe_count $stripe_count) roundrobin:" || + echo "Mkdir (stripe_count $stripe_count) on stripe $stripe_index" + +- stack_trap "unlinkmany -d $testdir/subdir $((100 * MDSCOUNT))" +- for (( i = 0; i < 100 * MDSCOUNT; i++ )); do ++ stack_trap "unlinkmany -d $testdir/subdir $((total / stripe_count))" ++ for (( i = 0; i < total / stripe_count; i++ )); do + eval $mkdir_cmd $testdir/subdir$i || + error "$mkdir_cmd subdir$i failed" + done +@@ -25679,13 +25702,13 @@ test_qos_mkdir() { + count=$($LFS getdirstripe -i $testdir/* | grep -c "^$i$") + echo "$count directories created on MDT$i" + if $test_mkdir_rr; then +- (( $count == 100 )) || ++ (( count == total / stripe_count / MDSCOUNT )) || + error "subdirs are not evenly distributed" +- elif (( $i == $stripe_index )); then +- (( $count == 100 * MDSCOUNT )) || ++ elif (( i == stripe_index )); then ++ (( count == total / stripe_count )) || + error "$count subdirs created on MDT$i" + else +- (( $count == 0 )) || ++ (( count == 0 )) || + error "$count subdirs created on MDT$i" + fi + +@@ -25694,8 +25717,8 @@ test_qos_mkdir() { + grep -c -P "^\s+$i\t") + echo "$count stripes created on MDT$i" + # deviation should < 5% of average +- (( $count >= 95 * stripe_count && +- $count <= 105 * stripe_count)) || ++ delta=$((count - total / MDSCOUNT)) ++ (( ${delta#-} <= total / MDSCOUNT / 20 )) || + error "stripes are not evenly distributed" + fi + done +@@ -25730,11 +25753,13 @@ test_qos_mkdir() { + min_index=$i + fi + done ++ echo "stripe_count=$stripe_count min_idx=$min_index max_idx=$max_index" + +- (( ${ffree[min_index]} > 0 )) || +- skip "no free files in MDT$min_index" ++ (( min > 0 )) || skip "low space on MDT$min_index" + (( ${ffree[min_index]} < 10000000 )) || +- skip "too many free files in MDT$min_index" ++ skip "too many free files on MDT$min_index" ++ ++ generate_uneven_mdts 120 + + echo "MDT filesfree available: ${ffree[*]}" + echo "MDT blocks available: ${bavail[*]}" +@@ -25753,10 +25778,9 @@ test_qos_mkdir() { + sleep 1 + + testdir=$DIR/$tdir-s$stripe_count/qos +- local num=200 + +- stack_trap "unlinkmany -d $testdir/subdir $((num * MDSCOUNT))" +- for (( i = 0; i < num * MDSCOUNT; i++ )); do ++ stack_trap "unlinkmany -d $testdir/subdir $((total / stripe_count))" ++ for (( i = 0; i < total / stripe_count; i++ )); do + eval $mkdir_cmd $testdir/subdir$i || + error "$mkdir_cmd subdir$i failed" + done +@@ -25765,14 +25789,14 @@ test_qos_mkdir() { + for (( i = 0; i < $MDSCOUNT; i++ )); do + count=$($LFS getdirstripe -i $testdir/* | grep -c "^$i$") + (( count > max )) && max=$count +- echo "$count directories created on MDT$i" ++ echo "$count directories created on MDT$i : curmax=$max" + done + + min=$($LFS getdirstripe -i $testdir/* | grep -c "^$min_index$") + +- # D-value should > 10% of averge +- (( max - min > num / 10 )) || +- error "subdirs shouldn't be evenly distributed: $max - $min < $((num / 10))" ++ # D-value should > 10% of average ++ (( max - min > total / stripe_count / MDSCOUNT / 10 )) || ++ error "subdirs shouldn't be evenly distributed: $max - $min <= $((total / stripe_count / MDSCOUNT / 10))" + + # ditto for stripes + if (( stripe_count > 1 )); then +@@ -25786,8 +25810,8 @@ test_qos_mkdir() { + + min=$($LFS getdirstripe $testdir/* | + grep -c -P "^\s+$min_index\t") +- (( max - min > num * stripe_count / 10 )) || +- error "stripes shouldn't be evenly distributed: $max - $min < $((num / 10)) * $stripe_count" ++ (( max - min > total / MDSCOUNT / 10 )) || ++ error "stripes shouldn't be evenly distributed: $max - $min <= $((total / MDSCOUNT / 10))" + fi + } + +@@ -25820,10 +25844,23 @@ test_413a() { + [ $MDS1_VERSION -lt $(version_code 2.12.52) ] && + skip "Need server version at least 2.12.52" + ++ local stripe_max=$((MDSCOUNT - 1)) + local stripe_count + +- generate_uneven_mdts 100 +- for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do ++ # let caller set maxage for latest result ++ set_maxage 1 ++ ++ # fill MDT unevenly ++ generate_uneven_mdts 120 ++ ++ # test 4-stripe directory at most, otherwise it's too slow ++ # We are being very defensive. Although Autotest uses 4 MDTs. ++ # We make sure stripe_max does not go over 4. ++ (( stripe_max > 4 )) && stripe_max=4 ++ # unlinking striped directory is slow on zfs, and may timeout, only test ++ # plain directory ++ [ "$mds1_FSTYPE" == "zfs" ] && stripe_max=1 ++ for stripe_count in $(seq 1 $stripe_max); do + mkdir $DIR/$tdir-s$stripe_count || error "mkdir failed" + mkdir $DIR/$tdir-s$stripe_count/rr || error "mkdir failed" + $LFS mkdir -i $(most_full_mdt) $DIR/$tdir-s$stripe_count/qos || +@@ -25840,11 +25877,22 @@ test_413b() { + [ $MDS1_VERSION -lt $(version_code 2.12.52) ] && + skip "Need server version at least 2.12.52" + ++ local stripe_max=$((MDSCOUNT - 1)) + local testdir + local stripe_count + +- generate_uneven_mdts 100 +- for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do ++ # let caller set maxage for latest result ++ set_maxage 1 ++ ++ # fill MDT unevenly ++ generate_uneven_mdts 120 ++ ++ # test 4-stripe directory at most, otherwise it's too slow ++ # We are being very defensive. Although Autotest uses 4 MDTs. ++ # We make sure stripe_max does not go over 4. ++ (( stripe_max > 4 )) && stripe_max=4 ++ [ "$mds1_FSTYPE" == "zfs" ] && stripe_max=1 ++ for stripe_count in $(seq 1 $stripe_max); do + testdir=$DIR/$tdir-s$stripe_count + mkdir $testdir || error "mkdir $testdir failed" + mkdir $testdir/rr || error "mkdir rr failed" +@@ -25869,6 +25917,23 @@ test_413c() { + local testdir + local inherit + local inherit_rr ++ local lmv_qos_maxage ++ local lod_qos_maxage ++ ++ # let caller set maxage for latest result ++ lmv_qos_maxage=$($LCTL get_param -n lmv.*.qos_maxage) ++ $LCTL set_param lmv.*.qos_maxage=1 ++ stack_trap "$LCTL set_param \ ++ lmv.*.qos_maxage=$lmv_qos_maxage > /dev/null" RETURN ++ lod_qos_maxage=$(do_facet mds1 $LCTL get_param -n \ ++ lod.$FSNAME-MDT0000-mdtlov.qos_maxage | awk '{ print $1 }') ++ do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \ ++ lod.*.mdt_qos_maxage=1 ++ stack_trap "do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \ ++ lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null" RETURN ++ ++ # fill MDT unevenly ++ generate_uneven_mdts 120 + + testdir=$DIR/${tdir}-s1 + mkdir $testdir || error "mkdir $testdir failed" +@@ -26019,13 +26084,15 @@ test_413z() { + local pid + + for subdir in $(\ls -1 -d $DIR/d413*-fillmdt/*); do +- unlinkmany $subdir/f. 1000 & ++ unlinkmany $subdir/f. $TEST413_COUNT & + pids="$pids $!" + done + + for pid in $pids; do + wait $pid + done ++ ++ true + } + run_test 413z "413 test cleanup" + +diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh +index 6126541ef4..d01bfdd3b3 100755 +--- a/lustre/tests/test-framework.sh ++++ b/lustre/tests/test-framework.sh +@@ -10634,7 +10634,7 @@ function createmany() { + fi + $LUSTRE/tests/createmany $* + rc=$? +- debugrestore ++ debugrestore > /dev/null + + return $rc + } +@@ -10649,26 +10649,68 @@ function unlinkmany() { + fi + $LUSTRE/tests/unlinkmany $* + rc=$? +- debugrestore ++ debugrestore > /dev/null + + return $rc + } + ++# Check if fallocate on facet is working. Returns fallocate mode if enabled. ++# Takes optional facet name as argument, to allow separate MDS/OSS checks. ++function check_fallocate_supported() ++{ ++ local facet=${1:-ost1} ++ local supported="FALLOCATE_SUPPORTED_$facet" ++ local fstype="${facet}_FSTYPE" ++ ++ if [[ -n "${!supported}" ]]; then ++ echo "${!supported}" ++ return 0 ++ fi ++ if [[ -z "${!fstype}" ]]; then ++ eval export $fstype=$(facet_fstype $facet) ++ fi ++ if [[ "${!fstype}" != "ldiskfs" ]]; then ++ echo "fallocate on ${!fstype} doesn't consume space" 1>&2 ++ return 1 ++ fi ++ ++ local fa_mode="osd-ldiskfs.$(facet_svc $facet).fallocate_zero_blocks" ++ local mode=$(do_facet $facet $LCTL get_param -n $fa_mode 2>/dev/null | ++ head -n 1) ++ ++ if [[ -z "$mode" ]]; then ++ echo "fallocate not supported on $facet" 1>&2 ++ return 1 ++ fi ++ eval export $supported="$mode" ++ ++ echo ${!supported} ++ return 0 ++} ++ ++# Check if fallocate supported on OSTs, enable if unset, skip if unavailable. ++# Takes optional facet name as argument. ++function check_fallocate_or_skip() ++{ ++ local facet=$1 ++ ++ check_fallocate_supported $1 || skip "fallocate not supported" ++} ++ + # Check if fallocate supported on OSTs, enable if unset, default mode=0 + # Optionally pass the OST fallocate mode (0=unwritten extents, 1=zero extents) + function check_set_fallocate() + { + local new_mode="$1" +- local osts=$(comma_list $(osts_nodes)) + local fa_mode="osd-ldiskfs.*.fallocate_zero_blocks" +- local old_mode=$(do_facet ost1 $LCTL get_param -n $fa_mode 2>/dev/null| +- head -n 1) ++ local old_mode="$(check_fallocate_supported)" + + [[ -n "$old_mode" ]] || { echo "fallocate not supported"; return 1; } + [[ -z "$new_mode" && "$old_mode" != "-1" ]] && + { echo "keep default fallocate mode: $old_mode"; return 0; } + [[ "$new_mode" && "$old_mode" == "$new_mode" ]] && + { echo "keep current fallocate mode: $old_mode"; return 0; } ++ local osts=$(comma_list $(osts_nodes)) + + stack_trap "do_nodes $osts $LCTL set_param $fa_mode=$old_mode" + do_nodes $osts $LCTL set_param $fa_mode=${new_mode:-0} || +@@ -10678,8 +10720,7 @@ function check_set_fallocate() + # Check if fallocate supported on OSTs, enable if unset, skip if unavailable + function check_set_fallocate_or_skip() + { +- [ "$ost1_FSTYPE" != ldiskfs ] && skip "non-ldiskfs backend" +- check_set_fallocate || skip "need at least 2.13.57 for fallocate" ++ check_set_fallocate || skip "need >= 2.13.57 and ldiskfs for fallocate" + } + + function disable_opencache() +-- +2.33.0 + diff --git a/0006-LU-11785-tests-fix-conf-sanity-98-mount-check-on-64K.patch b/0006-LU-11785-tests-fix-conf-sanity-98-mount-check-on-64K.patch deleted file mode 100644 index 980a0af2d463943a0a94ce67e44c29ca47635809..0000000000000000000000000000000000000000 --- a/0006-LU-11785-tests-fix-conf-sanity-98-mount-check-on-64K.patch +++ /dev/null @@ -1,45 +0,0 @@ -From 3e9a06398a168a52f72a65450e5249ed502a86cd Mon Sep 17 00:00:00 2001 -From: Kevin Zhao -Date: Fri, 28 Oct 2022 10:05:24 +0800 -Subject: [PATCH 06/61] LU-11785 tests: fix conf-sanity/98 mount check on 64K - page - -This patch fix the mount option length check expectation -fail on 64K page. Since the maxopt_len is the minmium -value of page_size or 64K page_size, but the test cases -only hard code the length of option to the 4K one. This -patch add the mount options according to the page size. - -Lustre-change: https://review.whamcloud.com/48177 -Lustre-commit: 4068ca725954db2a1fc42bf8d184f4672c2ed113 - -Test-Parameters: trivial testlist=conf-sanity env=ONLY=98 -Test-Parameters: testlist=conf-sanity env=ONLY=98 clientarch=aarch64 clientdistro=el8.7 -Signed-off-by: Kevin Zhao -Change-Id: Icdeb8b73308056e216c3f4ce71907b0c928d2c30 -Reviewed-by: Andreas Dilger -Reviewed-by: xinliang -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51288 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/tests/conf-sanity.sh | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh -index dc00fb42ff..a52429afe0 100755 ---- a/lustre/tests/conf-sanity.sh -+++ b/lustre/tests/conf-sanity.sh -@@ -7783,7 +7783,7 @@ test_98() - setup - check_mount || error "mount failed" - mountopt="user_xattr" -- for ((x = 1; x <= 400; x++)); do -+ for ((x = 1; x <= PAGE_SIZE/11; x++)); do - mountopt="$mountopt,user_xattr" - done - mount_client $MOUNT remount,$mountopt 2>&1 | grep "too long" || --- -2.33.0 - diff --git a/0006-LU-15207-libcfs-reset-hs_rehash_bits.patch b/0006-LU-15207-libcfs-reset-hs_rehash_bits.patch new file mode 100644 index 0000000000000000000000000000000000000000..536c49c0984d623eb63bfc6ebd91a17f8a6207c0 --- /dev/null +++ b/0006-LU-15207-libcfs-reset-hs_rehash_bits.patch @@ -0,0 +1,50 @@ +From e063351ab068a795dadbefaa5514d5bf7b7c4eb2 Mon Sep 17 00:00:00 2001 +From: Alex Zhuravlev +Date: Thu, 11 Nov 2021 11:19:46 +0300 +Subject: [PATCH 06/49] LU-15207 libcfs: reset hs_rehash_bits + +if rehash work is cancelled, then nobody resets +hs_rehash_bits and the first iterator asserts +at LASSERT(!cfs_hash_is_rehashing(hs)) in +cfs_hash_for_each_relax(). + +Lustre-change: https://review.whamcloud.com/45533 +Lustre-commit: 9257f24dfdf9f0a68512fce52d79064f78d9dc88 + +Signed-off-by: Alex Zhuravlev +Change-Id: I1a567f6be77ca6c45e5d4f256722206b12588554 +Reviewed-by: Andreas Dilger +Reviewed-by: James Simmons +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53283 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + libcfs/libcfs/hash.c | 11 +++++++++-- + 1 file changed, 9 insertions(+), 2 deletions(-) + +diff --git a/libcfs/libcfs/hash.c b/libcfs/libcfs/hash.c +index ca234ef096..6fb32ad9cc 100644 +--- a/libcfs/libcfs/hash.c ++++ b/libcfs/libcfs/hash.c +@@ -1828,8 +1828,15 @@ EXPORT_SYMBOL(cfs_hash_for_each_key); + void + cfs_hash_rehash_cancel(struct cfs_hash *hs) + { +- LASSERT(cfs_hash_with_rehash(hs)); +- cancel_work_sync(&hs->hs_rehash_work); ++ LASSERT(hs->hs_iterators > 0 || hs->hs_exiting); ++ while (cfs_hash_is_rehashing(hs)) { ++ if (cancel_work_sync(&hs->hs_rehash_work)) { ++ cfs_hash_lock(hs, 1); ++ hs->hs_rehash_bits = 0; ++ cfs_hash_unlock(hs, 1); ++ } ++ cond_resched(); ++ } + } + + void +-- +2.33.0 + diff --git a/0007-LU-11388-tests-replay-single-131b-to-refresh-grants.patch b/0007-LU-11388-tests-replay-single-131b-to-refresh-grants.patch deleted file mode 100644 index a2734481af6e13d6620dabd48f8b4ff641479212..0000000000000000000000000000000000000000 --- a/0007-LU-11388-tests-replay-single-131b-to-refresh-grants.patch +++ /dev/null @@ -1,45 +0,0 @@ -From 653ae754fa93ecf8b9d290675122956eaf63b6af Mon Sep 17 00:00:00 2001 -From: Alex Zhuravlev -Date: Mon, 17 Apr 2023 21:13:59 +0300 -Subject: [PATCH 07/61] LU-11388 tests: replay-single/131b to refresh grants - -so that the write (to be replayed after replay-barrier) -doesn't turn sync due to insufficient grant. - -Lustre-change: https://review.whamcloud.com/50661 -Lustre-commit: 384e1e858eef826677bfa6913074a83c4fab37d3 - -Test-Parameters: trivial testlist=replay-single env=ONLY=131b,ONLY_REPEAT=30 -Fixes: cb3b2bb683 ("LU-11388 test: enable replay-single test_131b") -Signed-off-by: Alex Zhuravlev -Change-Id: If4656c1028b49c58eedd905abd0c329f3706f491 -Reviewed-by: Mikhail Pershin -Reviewed-by: Jian Yu -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51289 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/tests/replay-single.sh | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh -index a0ac037ead..c6641c0d0e 100755 ---- a/lustre/tests/replay-single.sh -+++ b/lustre/tests/replay-single.sh -@@ -4858,6 +4858,12 @@ test_131b() { - [ "$MDS1_VERSION" -lt $(version_code 2.10.90) ] && - skip "Do not support Data-on-MDT before 2.11" - -+ # refresh grants so write after replay_barrier doesn't -+ # turn sync -+ $LFS setstripe -E 1M -L mdt -E EOF -c 2 $DIR/$tfile-2 -+ stack_trap "rm -f $DIR/$tfile-2" -+ dd if=/dev/zero of=$DIR/$tfile-2 bs=64k count=2 || -+ error "can't dd" - $LFS setstripe -E 1M -L mdt -E EOF -c 2 $DIR/$tfile - replay_barrier $SINGLEMDS - echo "dom_data" | dd of=$DIR/$tfile bs=8 count=1 --- -2.33.0 - diff --git a/0007-LU-16076-utils-enhance-lfs-check-command.patch b/0007-LU-16076-utils-enhance-lfs-check-command.patch new file mode 100644 index 0000000000000000000000000000000000000000..cf30c9ca59ff53a485cecb50906192601340d4cf --- /dev/null +++ b/0007-LU-16076-utils-enhance-lfs-check-command.patch @@ -0,0 +1,422 @@ +From 7c9a1eaf6419ffadeebcbf21f2735ffda59f0694 Mon Sep 17 00:00:00 2001 +From: Lei Feng +Date: Mon, 8 Aug 2022 10:59:25 +0800 +Subject: [PATCH 07/49] LU-16076 utils: enhance 'lfs check' command + +Add optional argument to 'lfs check' command so that only the +servers related to the specified lustre file system is checked. + +Lustre-change: https://review.whamcloud.com/48155 +Lustre-commit: f5ca6853b8d8b918b0228af31fa8249be49d3000 + +Signed-off-by: Lei Feng +Test-Parameters: trivial testlist=sanityn env=ONLY=113 +Change-Id: I826a8e822af0a290f06ffaadadf1bb7f86899d99 +Reviewed-by: Andreas Dilger +Reviewed-by: Emoly Liu +Reviewed-by: Jian Yu +Signed-off-by: Etienne AUJAMES +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51365 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + lustre/doc/lfs.1 | 7 +-- + lustre/tests/sanityn.sh | 36 ++++++++++++++- + lustre/utils/lfs.c | 19 +++++--- + lustre/utils/liblustreapi.c | 76 ++++++++++++++++++++++++++----- + lustre/utils/liblustreapi_fid.c | 4 +- + lustre/utils/liblustreapi_hsm.c | 2 +- + lustre/utils/liblustreapi_pcc.c | 2 +- + lustre/utils/lustreapi_internal.h | 3 +- + 8 files changed, 123 insertions(+), 26 deletions(-) + +diff --git a/lustre/doc/lfs.1 b/lustre/doc/lfs.1 +index 1e3b9c1694..d5501661b2 100644 +--- a/lustre/doc/lfs.1 ++++ b/lustre/doc/lfs.1 +@@ -7,7 +7,7 @@ lfs \- client utility for Lustre-specific file layout and other attributes + .br + .B lfs changelog_clear <\fImdtname\fR> <\fIid\fR> <\fIendrec\fR> + .br +-.B lfs check \fR<\fBmgts\fR|\fBmdts\fR|\fBosts\fR|\fBall\fR> ++.B lfs check \fR<\fBmgts\fR|\fBmdts\fR|\fBosts\fR|\fBall\fR> [\fIpath\fR] + .br + .B lfs data_version \fR[\fB-nrw\fR] \fB<\fIfilename\fR> + .br +@@ -158,9 +158,10 @@ interest to a particular consumer , potentially allowing the MDT to + free up disk space. An of 0 indicates the current last record. + Changelog consumers must be registered on the MDT node using \fBlctl\fR. + .TP +-.B check ++.B check [path] + Display the status of the MGTs, MDTs or OSTs (as specified in the command) or +-all the servers (MGTs, MDTs and OSTs). ++all the servers (MGTs, MDTs and OSTs). If \fBpath\fR is provided, display ++the status of the lustre file system mounted at specified \fBpath\fR only. + .TP + .B data_version [-nrw] + Display the current version of file data. If -n is specified, the data version +diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh +index ca366fd10a..e7aaa9b175 100755 +--- a/lustre/tests/sanityn.sh ++++ b/lustre/tests/sanityn.sh +@@ -482,7 +482,7 @@ test_16d() { + + $LFS setstripe -c -1 $file1 # b=10919 + $LCTL set_param ldlm.namespaces.*.lru_size=clear +- ++ + # direct write on one client and direct read from another + dd if=/dev/urandom of=$file1 bs=1M count=100 oflag=direct + dd if=$file2 of=$tmpfile iflag=direct bs=1M +@@ -5932,6 +5932,40 @@ test_112() { + } + run_test 112 "update max-inherit in default LMV" + ++test_113 () { ++ (( MDS1_VERSION >= $(version_code 2.15.3) )) || ++ skip "Need server version at least 2.15.3" ++ ++ local instance ++ local nid ++ ++ instance=$($LFS getname -i $DIR1) || ++ error "cannot get instance of $DIR1" ++ ++ $LFS check osts $DIR1 | grep $instance || ++ error "cannot find OSTs of instance $instance" ++ ++ $LFS check osts $DIR1 | grep -v $instance ++ if (( $? == 0 )); then ++ error "find OSTs other than instance $instance" ++ fi ++ ++ $LFS check osts | grep $instance || ++ error "cannot find other OSTs" ++ ++ nid=$(df $DIR2 | tail -1 | sed 's%:/.*%%') || ++ error "cannot parse nid for $DIR2" ++ ++ $LFS check mgts $DIR2 | grep MGC$nid || ++ error "cannot find mgc of $nid" ++ ++ $LFS check mgts $DIR2 | grep -v MGC$nid ++ if (( $? == 0 )); then ++ error "find MGTs other than nid $nid" ++ fi ++} ++run_test 113 "check servers of specified fs" ++ + log "cleanup: ======================================================" + + # kill and wait in each test only guarentee script finish, but command in script +diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c +index 95780b9c23..ebe2fc823c 100644 +--- a/lustre/utils/lfs.c ++++ b/lustre/utils/lfs.c +@@ -429,8 +429,8 @@ command_t cmdlist[] = { + "\t ^: used before a flag indicates to exclude it\n"}, + {"check", lfs_check, 0, + "Display the status of MGTs, MDTs or OSTs (as specified in the command)\n" +- "or all the servers (MGTs, MDTs and OSTs).\n" +- "usage: check {mgts|osts|mdts|all}"}, ++ "or all the servers (MGTs, MDTs and OSTs) [for specified path only].\n" ++ "usage: check {mgts|osts|mdts|all} [path]"}, + {"osts", lfs_osts, 0, "list OSTs connected to client " + "[for specified path only]\n" "usage: osts [path]"}, + {"mdts", lfs_mdts, 0, "list MDTs connected to client " +@@ -7385,7 +7385,7 @@ static int lfs_getname(int argc, char **argv) + + static int lfs_check(int argc, char **argv) + { +- char mntdir[PATH_MAX] = {'\0'}; ++ char mntdir[PATH_MAX] = {'\0'}, path[PATH_MAX] = {'\0'}; + int num_types = 1; + char *obd_types[3]; + char obd_type1[4]; +@@ -7393,7 +7393,7 @@ static int lfs_check(int argc, char **argv) + char obd_type3[4]; + int rc; + +- if (argc != 2) { ++ if (argc < 2 || argc > 3) { + fprintf(stderr, "%s check: server type must be specified\n", + progname); + return CMD_HELP; +@@ -7422,7 +7422,14 @@ static int lfs_check(int argc, char **argv) + return CMD_HELP; + } + +- rc = llapi_search_mounts(NULL, 0, mntdir, NULL); ++ if (argc >= 3 && !realpath(argv[2], path)) { ++ rc = -errno; ++ fprintf(stderr, "error: invalid path '%s': %s\n", ++ argv[2], strerror(-rc)); ++ return rc; ++ } ++ ++ rc = llapi_search_mounts(path, 0, mntdir, NULL); + if (rc < 0 || mntdir[0] == '\0') { + fprintf(stderr, + "%s check: cannot find mounted Lustre filesystem: %s\n", +@@ -7430,7 +7437,7 @@ static int lfs_check(int argc, char **argv) + return rc; + } + +- rc = llapi_target_check(num_types, obd_types, mntdir); ++ rc = llapi_target_check(num_types, obd_types, path); + if (rc) + fprintf(stderr, "%s check: cannot check target '%s': %s\n", + progname, argv[1], strerror(-rc)); +diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c +index 29b1e4e7a4..8b596af375 100644 +--- a/lustre/utils/liblustreapi.c ++++ b/lustre/utils/liblustreapi.c +@@ -84,6 +84,8 @@ + #include "lstddef.h" + + #define FORMATTED_BUF_LEN 1024 ++#define MAX_LINE_LEN 256 ++#define MAX_INSTANCE_LEN 32 + + static int llapi_msg_level = LLAPI_MSG_MAX; + const char *liblustreapi_cmd; +@@ -1254,12 +1256,13 @@ static struct { + dev_t dev; + char fsname[PATH_MAX]; + char mnt_dir[PATH_MAX]; ++ char nid[MAX_LINE_LEN]; + } root_cached = { 0 }; + + static pthread_rwlock_t root_cached_lock = PTHREAD_RWLOCK_INITIALIZER; + + static int get_root_path_fast(int want, char *fsname, int *outfd, char *path, +- dev_t *dev) ++ dev_t *dev, char *nid) + { + int rc = -ENODEV; + int fsnamelen; +@@ -1298,14 +1301,15 @@ static int get_root_path_fast(int want, char *fsname, int *outfd, char *path, + *dev = root_cached.dev; + if ((want & WANT_FD) && outfd) + rc = get_root_fd(root_cached.mnt_dir, outfd); +- ++ if ((want & WANT_NID) && nid) ++ strcpy(nid, root_cached.nid); + out_unlock: + pthread_rwlock_unlock(&root_cached_lock); + return rc; + } + + static int get_root_path_slow(int want, char *fsname, int *outfd, char *path, +- int index, dev_t *dev) ++ int index, dev_t *dev, char *nid) + { + struct mntent mnt; + char buf[PATH_MAX]; +@@ -1392,6 +1396,10 @@ static int get_root_path_slow(int want, char *fsname, int *outfd, char *path, + strncpy(root_cached.mnt_dir, mnt.mnt_dir, mntlen); + root_cached.mnt_dir[mntlen] = '\0'; + root_cached.dev = devmnt; ++ ptr_end = strchr(mnt.mnt_fsname, ':'); ++ strncpy(root_cached.nid, mnt.mnt_fsname, ++ ptr_end - mnt.mnt_fsname); ++ root_cached.nid[ptr_end - mnt.mnt_fsname] = '\0'; + + pthread_rwlock_unlock(&root_cached_lock); + } +@@ -1408,6 +1416,11 @@ static int get_root_path_slow(int want, char *fsname, int *outfd, char *path, + *dev = devmnt; + if ((want & WANT_FD) && outfd) + rc = get_root_fd(mnt.mnt_dir, outfd); ++ if ((want & WANT_NID) && nid) { ++ ptr_end = strchr(mnt.mnt_fsname, ':'); ++ strncpy(nid, mnt.mnt_fsname, ptr_end - mnt.mnt_fsname); ++ nid[ptr_end - mnt.mnt_fsname] = '\0'; ++ } + + out: + endmntent(fp); +@@ -1419,14 +1432,15 @@ out: + * Either the fsname or path must not be NULL + */ + int get_root_path(int want, char *fsname, int *outfd, char *path, int index, +- dev_t *dev) ++ dev_t *dev, char *nid) + { + int rc = -ENODEV; + + if (!(want & WANT_INDEX)) +- rc = get_root_path_fast(want, fsname, outfd, path, dev); ++ rc = get_root_path_fast(want, fsname, outfd, path, dev, nid); + if (rc) +- rc = get_root_path_slow(want, fsname, outfd, path, index, dev); ++ rc = get_root_path_slow(want, fsname, outfd, path, index, dev, ++ nid); + + if (!rc || !(want & WANT_ERROR)) + return rc; +@@ -1468,7 +1482,7 @@ int llapi_search_mounts(const char *pathname, int index, char *mntdir, + + if (fsname) + want |= WANT_FSNAME; +- return get_root_path(want, fsname, NULL, mntdir, idx, NULL); ++ return get_root_path(want, fsname, NULL, mntdir, idx, NULL, NULL); + } + + /* Given a path, find the corresponding Lustre fsname */ +@@ -1501,7 +1515,7 @@ int llapi_search_fsname(const char *pathname, char *fsname) + } + + rc = get_root_path(WANT_FSNAME | WANT_ERROR, fsname, NULL, NULL, -1, +- &dev); ++ &dev, NULL); + + return rc; + } +@@ -1517,7 +1531,7 @@ int llapi_search_rootpath(char *pathname, const char *fsname) + */ + pathname[0] = 0; + return get_root_path(WANT_PATH, (char *)fsname, NULL, pathname, -1, +- NULL); ++ NULL, NULL); + } + + int llapi_search_rootpath_by_dev(char *pathname, dev_t dev) +@@ -1530,7 +1544,7 @@ int llapi_search_rootpath_by_dev(char *pathname, dev_t dev) + * clear it for safety + */ + pathname[0] = 0; +- return get_root_path(WANT_PATH, NULL, NULL, pathname, -1, &dev); ++ return get_root_path(WANT_PATH, NULL, NULL, pathname, -1, &dev, NULL); + } + + /** +@@ -6706,10 +6720,27 @@ free_path: + return rc; + } + ++struct check_target_filter { ++ char *nid; ++ char *instance; ++}; ++ + static void do_target_check(char *obd_type_name, char *obd_name, + char *obd_uuid, void *args) + { + int rc; ++ struct check_target_filter *filter = args; ++ ++ if (filter != NULL) { ++ /* check nid if obd type is mgc */ ++ if (strcmp(obd_type_name, "mgc") == 0) { ++ if (strcmp(obd_name + 3, filter->nid) != 0) ++ return; ++ } ++ /* check instance for other types of device (osc/mdc) */ ++ else if (strstr(obd_name, filter->instance) == NULL) ++ return; ++ } + + rc = llapi_ping(obd_type_name, obd_name); + if (rc == ENOTCONN) +@@ -6722,7 +6753,30 @@ static void do_target_check(char *obd_type_name, char *obd_name, + + int llapi_target_check(int type_num, char **obd_type, char *dir) + { +- return llapi_target_iterate(type_num, obd_type, NULL, do_target_check); ++ char nid[MAX_LINE_LEN], instance[MAX_INSTANCE_LEN]; ++ struct check_target_filter filter = {NULL, NULL}; ++ int rc; ++ ++ if (dir == NULL || dir[0] == '\0') ++ return llapi_target_iterate(type_num, obd_type, NULL, ++ do_target_check); ++ ++ rc = get_root_path(WANT_NID | WANT_ERROR, NULL, NULL, dir, -1, NULL, ++ nid); ++ if (rc) { ++ llapi_error(LLAPI_MSG_ERROR, rc, ++ "cannot get nid of path '%s'", dir); ++ return rc; ++ } ++ filter.nid = nid; ++ ++ rc = llapi_get_instance(dir, instance, ARRAY_SIZE(instance)); ++ if (rc) ++ return rc; ++ filter.instance = instance; ++ ++ return llapi_target_iterate(type_num, obd_type, &filter, ++ do_target_check); + } + + #undef MAX_STRING_SIZE +diff --git a/lustre/utils/liblustreapi_fid.c b/lustre/utils/liblustreapi_fid.c +index 8d3f2686d7..cc76ae6d88 100644 +--- a/lustre/utils/liblustreapi_fid.c ++++ b/lustre/utils/liblustreapi_fid.c +@@ -224,10 +224,10 @@ int llapi_fid2path(const char *path_or_device, const char *fidstr, char *path, + + if (*path_or_device == '/') + rc = get_root_path(WANT_FD, NULL, &mnt_fd, +- (char *)path_or_device, -1, NULL); ++ (char *)path_or_device, -1, NULL, NULL); + else + rc = get_root_path(WANT_FD, (char *)path_or_device, +- &mnt_fd, NULL, -1, NULL); ++ &mnt_fd, NULL, -1, NULL, NULL); + + if (rc < 0) + goto out; +diff --git a/lustre/utils/liblustreapi_hsm.c b/lustre/utils/liblustreapi_hsm.c +index 052e6cd446..e6342939bb 100644 +--- a/lustre/utils/liblustreapi_hsm.c ++++ b/lustre/utils/liblustreapi_hsm.c +@@ -1592,7 +1592,7 @@ int llapi_hsm_request(const char *path, const struct hsm_user_request *request) + int rc; + int fd; + +- rc = get_root_path(WANT_FD, NULL, &fd, (char *)path, -1, NULL); ++ rc = get_root_path(WANT_FD, NULL, &fd, (char *)path, -1, NULL, NULL); + if (rc) + return rc; + +diff --git a/lustre/utils/liblustreapi_pcc.c b/lustre/utils/liblustreapi_pcc.c +index b1652c238b..a2ed7665f1 100644 +--- a/lustre/utils/liblustreapi_pcc.c ++++ b/lustre/utils/liblustreapi_pcc.c +@@ -212,7 +212,7 @@ int llapi_pcc_detach_fid(const char *mntpath, const struct lu_fid *fid, + int fd; + struct lu_pcc_detach_fid detach; + +- rc = get_root_path(WANT_FD, NULL, &fd, (char *)mntpath, -1, NULL); ++ rc = get_root_path(WANT_FD, NULL, &fd, (char *)mntpath, -1, NULL, NULL); + if (rc) { + llapi_error(LLAPI_MSG_ERROR, rc, "cannot get root path: %s", + mntpath); +diff --git a/lustre/utils/lustreapi_internal.h b/lustre/utils/lustreapi_internal.h +index 41b7087480..502fb8b911 100644 +--- a/lustre/utils/lustreapi_internal.h ++++ b/lustre/utils/lustreapi_internal.h +@@ -51,6 +51,7 @@ + #define WANT_INDEX 0x8 + #define WANT_ERROR 0x10 + #define WANT_DEV 0x20 ++#define WANT_NID 0x40 + + /* Define a fixed 4096-byte encryption unit size */ + #define LUSTRE_ENCRYPTION_BLOCKBITS 12 +@@ -63,7 +64,7 @@ + #endif + + int get_root_path(int want, char *fsname, int *outfd, char *path, int index, +- dev_t *dev); ++ dev_t *dev, char *nid); + int llapi_ioctl_pack(struct obd_ioctl_data *data, char **pbuf, int max_len); + int llapi_ioctl_unpack(struct obd_ioctl_data *data, char *pbuf, int max_len); + int sattr_cache_get_defaults(const char *const fsname, +-- +2.33.0 + diff --git a/0008-LU-16057-obdclass-set-OBD_MD_FLGROUP-for-ladvise-RPC.patch b/0008-LU-16057-obdclass-set-OBD_MD_FLGROUP-for-ladvise-RPC.patch new file mode 100644 index 0000000000000000000000000000000000000000..cddd6249fd0d181e485c22821fa19e6720ea3904 --- /dev/null +++ b/0008-LU-16057-obdclass-set-OBD_MD_FLGROUP-for-ladvise-RPC.patch @@ -0,0 +1,79 @@ +From 6b7af79f6d2ea487e2af96fcb84b83b275ac9f8c Mon Sep 17 00:00:00 2001 +From: Li Dongyang +Date: Fri, 29 Jul 2022 16:35:41 +1000 +Subject: [PATCH 08/49] LU-16057 obdclass: set OBD_MD_FLGROUP for ladvise RPC + +ladvise RPC doesn't have OBD_MD_FLGROUP set, when RPC +reaches server, tgt_validate_obdo() will corrupt the FID +if it's seq is in FID_SEQ_NORMAL range. + +Do not mess with seq in obdo_to_ioobj() and tgt_validate_obdo(), +since 2.0 all RPCs should have OBD_MD_FLGROUP set. + +Add OBD_MD_FLGROUP for ladvise RPC to fix new client talking +to old servers. + +Lustre-change: https://review.whamcloud.com/48080 +Lustre-commit: bee803c6e440ba6b55e0ff356e5324f44cfa63eb + +Change-Id: I373b7f32458b18e29d9bb716a912fe4a54eccac5 +Signed-off-by: Li Dongyang +Reviewed-by: Andreas Dilger +Reviewed-by: Arshad Hussain +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53501 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + lustre/obdclass/obdo.c | 3 --- + lustre/osc/osc_io.c | 2 +- + lustre/target/tgt_handler.c | 5 +---- + 3 files changed, 2 insertions(+), 8 deletions(-) + +diff --git a/lustre/obdclass/obdo.c b/lustre/obdclass/obdo.c +index d17d19741b..b1d7337b94 100644 +--- a/lustre/obdclass/obdo.c ++++ b/lustre/obdclass/obdo.c +@@ -164,9 +164,6 @@ EXPORT_SYMBOL(obdo_cpy_md); + void obdo_to_ioobj(const struct obdo *oa, struct obd_ioobj *ioobj) + { + ioobj->ioo_oid = oa->o_oi; +- if (unlikely(!(oa->o_valid & OBD_MD_FLGROUP))) +- ostid_set_seq_mdt0(&ioobj->ioo_oid); +- + /* + * Since 2.4 this does not contain o_mode in the low 16 bits. + * Instead, it holds (bd_md_max_brw - 1) for multi-bulk BRW RPCs +diff --git a/lustre/osc/osc_io.c b/lustre/osc/osc_io.c +index 86399655f5..cd2a97fd82 100644 +--- a/lustre/osc/osc_io.c ++++ b/lustre/osc/osc_io.c +@@ -1049,7 +1049,7 @@ static int osc_io_ladvise_start(const struct lu_env *env, + + memset(oa, 0, sizeof(*oa)); + oa->o_oi = loi->loi_oi; +- oa->o_valid = OBD_MD_FLID; ++ oa->o_valid = OBD_MD_FLID | OBD_MD_FLGROUP; + obdo_set_parent_fid(oa, lio->li_fid); + + ladvise = ladvise_hdr->lah_advise; +diff --git a/lustre/target/tgt_handler.c b/lustre/target/tgt_handler.c +index 6be647b20c..a99436d695 100644 +--- a/lustre/target/tgt_handler.c ++++ b/lustre/target/tgt_handler.c +@@ -157,11 +157,8 @@ int tgt_validate_obdo(struct tgt_session_info *tsi, struct obdo *oa) + + /* Note: this check might be forced in 2.5 or 2.6, i.e. + * all of the requests are required to setup FLGROUP */ +- if (unlikely(!(oa->o_valid & OBD_MD_FLGROUP))) { +- ostid_set_seq_mdt0(oi); ++ if (unlikely(!(oa->o_valid & OBD_MD_FLGROUP))) + oa->o_valid |= OBD_MD_FLGROUP; +- seq = ostid_seq(oi); +- } + + if (unlikely(!(fid_seq_is_idif(seq) || fid_seq_is_mdt0(seq) || + fid_seq_is_norm(seq) || fid_seq_is_echo(seq)))) +-- +2.33.0 + diff --git a/0008-LU-16163-tests-skip-racer_on_nfs-for-NFSv3.patch b/0008-LU-16163-tests-skip-racer_on_nfs-for-NFSv3.patch deleted file mode 100644 index c683246dcfa46b57cfcffeaf9d15882f60eea636..0000000000000000000000000000000000000000 --- a/0008-LU-16163-tests-skip-racer_on_nfs-for-NFSv3.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 3626be5686cc395ce622d281a993603dba16e3e2 Mon Sep 17 00:00:00 2001 -From: Alex Deiter -Date: Fri, 7 Apr 2023 23:49:23 +0400 -Subject: [PATCH 08/61] LU-16163 tests: skip racer_on_nfs for NFSv3 - -Export ALWAYS_EXCEPT env for child NFS test - -Lustre-change: https://review.whamcloud.com/50579 -Lustre-commit: 892d726f274c7cd4e505689ad69194ac68dc323b - -Fixes: 513eb670b0 ("LU-16163 tests: skip racer_on_nfs for NFSv3") -Test-Parameters: trivial testlist=parallel-scale-nfsv3 -Signed-off-by: Alex Deiter -Change-Id: Ibb4a9916166f13ab9bd2374b33d4313453972276 -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51282 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Andreas Dilger -Reviewed-by: Oleg Drokin ---- - lustre/tests/parallel-scale-nfsv3.sh | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/lustre/tests/parallel-scale-nfsv3.sh b/lustre/tests/parallel-scale-nfsv3.sh -index 014d2cda2a..fdbfc949c8 100755 ---- a/lustre/tests/parallel-scale-nfsv3.sh -+++ b/lustre/tests/parallel-scale-nfsv3.sh -@@ -4,4 +4,9 @@ LUSTRE=${LUSTRE:-$(dirname $0)/..} - . $LUSTRE/tests/test-framework.sh - init_test_env $@ - --sh $LUSTRE/tests/parallel-scale-nfs.sh 3 -+export ALWAYS_EXCEPT="$PARALLEL_SCALE_NFSV3_EXCEPT " -+# Bug number for skipped test: LU-16163 -+ALWAYS_EXCEPT+=" racer_on_nfs " -+# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! -+ -+$LUSTRE/tests/parallel-scale-nfs.sh 3 --- -2.33.0 - diff --git a/0009-LU-14294-tests-fixed-NFS-configuration-issue.patch b/0009-LU-14294-tests-fixed-NFS-configuration-issue.patch deleted file mode 100644 index 8aa1834bb0370423cc46628d83478a60e6f00c0e..0000000000000000000000000000000000000000 --- a/0009-LU-14294-tests-fixed-NFS-configuration-issue.patch +++ /dev/null @@ -1,197 +0,0 @@ -From cef89c354f22f873f1f2e09536de7c690852828b Mon Sep 17 00:00:00 2001 -From: Alex Deiter -Date: Mon, 7 Nov 2022 21:47:21 +0400 -Subject: [PATCH 09/61] LU-14294 tests: fixed NFS configuration issue - -* Used the systemctl command to manage system services -* Used the same order of parameters to setup and cleanup NFS -* Used tab for indentation - -Lustre-change: https://review.whamcloud.com/49062 -Lustre-commit: 1a8fe55b17ac2bc2195aaba446467ccdac67b564 - -Test-Parameters: trivial clientdistro=el7.9 \ -testlist=parallel-scale-nfsv3,parallel-scale-nfsv4 -Test-Parameters: clientdistro=el8.7 \ -testlist=parallel-scale-nfsv3,parallel-scale-nfsv4 -Test-Parameters: clientdistro=el9.0 \ -testlist=parallel-scale-nfsv3,parallel-scale-nfsv4 -Test-Parameters: clientdistro=sles12sp5 \ -testlist=parallel-scale-nfsv3,parallel-scale-nfsv4 -Test-Parameters: clientdistro=sles15sp4 \ -testlist=parallel-scale-nfsv3,parallel-scale-nfsv4 -Test-Parameters: clientdistro=ubuntu2004 \ -testlist=parallel-scale-nfsv3,parallel-scale-nfsv4 - -Change-Id: I6b087035ac7524aa99c0facad48f8c3fb7444cbc -Signed-off-by: Alex Deiter -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51283 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Andreas Dilger -Reviewed-by: Oleg Drokin ---- - lustre/tests/parallel-scale-nfs.sh | 11 ++- - lustre/tests/setup-nfs.sh | 107 +++++++++++------------------ - 2 files changed, 47 insertions(+), 71 deletions(-) - -diff --git a/lustre/tests/parallel-scale-nfs.sh b/lustre/tests/parallel-scale-nfs.sh -index 7f426ee0a2..ab88f379a4 100755 ---- a/lustre/tests/parallel-scale-nfs.sh -+++ b/lustre/tests/parallel-scale-nfs.sh -@@ -47,7 +47,8 @@ cleanup_exit () { - } - - cleanup () { -- cleanup_nfs "$NFS_CLIMNTPT" "$LUSTRE_CLIENT_NFSSRV" "$NFS_CLIENTS" || -+ cleanup_nfs "$LUSTRE_CLIENT_NFSSRV" "$NFS_SRVMNTPT" \ -+ "$NFS_CLIENTS" "$NFS_CLIMNTPT" || \ - error_noexit false "failed to cleanup nfs" - zconf_umount $LUSTRE_CLIENT_NFSSRV $NFS_SRVMNTPT force || - error_noexit false "failed to umount lustre on"\ -@@ -63,8 +64,8 @@ zconf_mount $LUSTRE_CLIENT_NFSSRV $NFS_SRVMNTPT "$cl_mnt_opt" || - error "mount lustre on $LUSTRE_CLIENT_NFSSRV failed" - - # setup the nfs --setup_nfs "$NFSVERSION" "$NFS_SRVMNTPT" "$LUSTRE_CLIENT_NFSSRV" \ -- "$NFS_CLIENTS" "$NFS_CLIMNTPT" || -+setup_nfs "$LUSTRE_CLIENT_NFSSRV" "$NFS_SRVMNTPT" "$NFS_CLIENTS" \ -+ "$NFS_CLIMNTPT" "$NFSVERSION" || \ - error false "setup nfs failed!" - - NFSCLIENT=true -@@ -101,6 +102,10 @@ MPI_RUNAS=${MPI_RUNAS:-"runas -u $MPI_USER_UID -g $MPI_USER_GID"} - $GSS_KRB5 && refresh_krb5_tgt $MPI_USER_UID $MPI_USER_GID $MPI_RUNAS - - test_compilebench() { -+ if [[ "$TESTSUITE" =~ "parallel-scale-nfs" ]]; then -+ skip "LU-12957 and LU-13068: compilebench for $TESTSUITE" -+ fi -+ - run_compilebench $TESTDIR - } - run_test compilebench "compilebench" -diff --git a/lustre/tests/setup-nfs.sh b/lustre/tests/setup-nfs.sh -index ab3afff36c..eb6720febd 100755 ---- a/lustre/tests/setup-nfs.sh -+++ b/lustre/tests/setup-nfs.sh -@@ -1,77 +1,48 @@ - #!/bin/bash --#set -x --EXPORT_OPTS=${EXPORT_OPTS:-"rw,async,no_root_squash"} - --setup_nfs() { -- local NFS_VER=${1} -- local MNTPNT=${2} -- local LUSTRE_CLIENT=${3} -- local NFS_CLIENTS=${4} -- local nfs_climntpt=${5:-$MNTPNT} -- -- local export_opts_v=$EXPORT_OPTS -- -- echo "Exporting Lustre filesystem..." -- -- if [ "$NFS_VER" = "4" ]; then -- export_opts_v="$EXPORT_OPTS,fsid=0" -- do_nodes $LUSTRE_CLIENT "mkdir -p /var/lib/nfs/v4recovery" -- fi -- -- do_nodes $LUSTRE_CLIENT,$NFS_CLIENTS "grep -q rpc_pipefs' ' /proc/mounts ||\ -- { mkdir -p /var/lib/nfs/rpc_pipefs && \ -- mount -t rpc_pipefs sunrpc /var/lib/nfs/rpc_pipefs; }" || return 1 -- sleep 5 -- -- # get rid of old $MNTPNT entries in /etc/exports -- do_nodes $LUSTRE_CLIENT "sed -i '/${MNTPNT##*/}/d' /etc/exports && -- echo $MNTPNT *\($export_opts_v\) >> /etc/exports" || -- return 1 -- -- # restart nfs server according to distro -- do_nodes $LUSTRE_CLIENT "{ [[ -e /etc/SuSE-release ]] && -- service nfsserver restart; } || -- service nfs restart || -- service nfs-server restart" || return 1 -+DEFAULT_NFS_OPTIONS=${DEFAULT_NFS_OPTIONS:-"rw,async,no_root_squash"} -+DEFAULT_EXPORTS_FILE=${DEFAULT_EXPORTS_FILE:-"/etc/exports.d/lustre.exports"} - -- do_nodes $NFS_CLIENTS "chkconfig --list rpcidmapd 2>/dev/null | -- grep -q rpcidmapd && service rpcidmapd restart || -- true" -- -- echo -e "\nMounting NFS clients (version $NFS_VER)..." -+setup_nfs() { -+ local LUSTRE_CLIENT=$1 -+ local LUSTRE_MOUNT_POINT=$2 -+ local NFS_CLIENTS=$3 -+ local NFS_MOUNT_POINT=$4 -+ local NFS_VERSION=$5 -+ local EXPORTS_FILE=$DEFAULT_EXPORTS_FILE -+ local NFS_OPTIONS=$DEFAULT_NFS_OPTIONS -+ -+ echo "Exporting Lustre filesystem via NFS version $NFS_VERSION" -+ do_nodes "$LUSTRE_CLIENT" \ -+ "echo '$LUSTRE_MOUNT_POINT *($NFS_OPTIONS)' | \ -+ tee $EXPORTS_FILE" || return 1 -+ do_nodes "$LUSTRE_CLIENT" "systemctl restart nfs-server" || return 1 -+ do_nodes "$LUSTRE_CLIENT" "systemctl restart nfs-idmapd" || return 1 -+ -+ echo "Mounting NFS clients version $NFS_VERSION" -+ do_nodes "$NFS_CLIENTS" "systemctl restart nfs-idmapd" || return 1 -+ do_nodes "$NFS_CLIENTS" "mkdir -p $NFS_MOUNT_POINT" || return 1 -+ do_nodes "$NFS_CLIENTS" \ -+ "mount -v -t nfs -o nfsvers=$NFS_VERSION,async \ -+ $LUSTRE_CLIENT:$LUSTRE_MOUNT_POINT \ -+ $NFS_MOUNT_POINT" || return 1 - -- do_nodes $NFS_CLIENTS "mkdir -p $nfs_climntpt" || return 1 -- if [ "$NFS_VER" = "4" ]; then -- do_nodes $NFS_CLIENTS \ -- "mount -t nfs$NFS_VER -o async \ -- $LUSTRE_CLIENT:/ $nfs_climntpt" || return 1 -- else -- do_nodes $NFS_CLIENTS \ -- "mount -t nfs -o nfsvers=$NFS_VER,async \ -- $LUSTRE_CLIENT:$MNTPNT $nfs_climntpt" || return 1 -- fi - return 0 - } - - cleanup_nfs() { -- local MNTPNT=${1} -- local LUSTRE_CLIENT=${2} -- local NFS_CLIENTS=${3} -- -- echo -e "\nUnmounting NFS clients..." -- do_nodes $NFS_CLIENTS "umount -f $MNTPNT" || true -- -- echo -e "\nUnexporting Lustre filesystem..." -- do_nodes $NFS_CLIENTS "chkconfig --list rpcidmapd 2>/dev/null | -- grep -q rpcidmapd && service rpcidmapd stop || -- true" -- -- do_nodes $LUSTRE_CLIENT "{ [[ -e /etc/SuSE-release ]] && -- service nfsserver stop; } || -- service nfs stop || -- service nfs-server stop" || return 1 -- -- do_nodes $LUSTRE_CLIENT "sed -i '/${MNTPNT##*/}/d' /etc/exports" || return 1 -- -- do_nodes $LUSTRE_CLIENT "exportfs -v" -+ local LUSTRE_CLIENT=$1 -+ local LUSTRE_MOUNT_POINT=$2 -+ local NFS_CLIENTS=$3 -+ local NFS_MOUNT_POINT=$4 -+ local EXPORTS_FILE=$DEFAULT_EXPORTS_FILE -+ -+ echo "Unmounting NFS clients" -+ do_nodes "$NFS_CLIENTS" "umount -v -f $NFS_MOUNT_POINT" || return 1 -+ do_nodes "$NFS_CLIENTS" "systemctl stop nfs-idmapd" || return 1 -+ -+ echo "Unexporting Lustre filesystem" -+ do_nodes "$LUSTRE_CLIENT" "systemctl stop nfs-server" || return 1 -+ do_nodes "$LUSTRE_CLIENT" "systemctl stop nfs-idmapd" || return 1 -+ do_nodes "$LUSTRE_CLIENT" "rm -v $EXPORTS_FILE" || return 1 - } --- -2.33.0 - diff --git a/0009-LU-16772-quota-protect-lqe_glbl_data-in-qmt_site_rec.patch b/0009-LU-16772-quota-protect-lqe_glbl_data-in-qmt_site_rec.patch new file mode 100644 index 0000000000000000000000000000000000000000..f69557fc204c6a7596479633c86f805cc3eb1ea7 --- /dev/null +++ b/0009-LU-16772-quota-protect-lqe_glbl_data-in-qmt_site_rec.patch @@ -0,0 +1,214 @@ +From 197e9aa693ed6eaef7b358a8f4afca4e50e80bd2 Mon Sep 17 00:00:00 2001 +From: Sergey Cheremencev +Date: Tue, 25 Apr 2023 22:10:21 +0400 +Subject: [PATCH 09/49] LU-16772 quota: protect lqe_glbl_data in + qmt_site_recalc_cb + +lqe_glbl_data should be protected with lqe_glbl_data_lock in +qmt_site_recalc_sb like it did in other places. Otherwise it +may cause following panic: + + BUG: unable to handle kernel NULL pointer at 00000000000000f8 + qmt_site_recalc_cb+0x2f8/0x790 [lquota] + cfs_hash_for_each_tight+0x121/0x310 [libcfs] + qmt_pool_recalc+0x372/0x9f0 [lquota] + +Also protect lqe_glbl_data access with lqe_glbl_data_lock in +qmt_lvbo_free(). + +Lustre-change: https://review.whamcloud.com/50748 +Lustre-commit: 50ff4d1da63e8bc1dba4b6b52219fb7024f8d66f + +Fixes: 1dbcbd70f8 ("LU-15021 quota: protect lqe_glbl_data in lqe") +Signed-off-by: Sergey Cheremencev +Change-Id: I030f14b02062151f1708a03ac7414a9991f798f6 +Reviewed-by: Oleg Drokin +Signed-off-by: Etienne AUJAMES +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53284 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Andreas Dilger +--- + lustre/quota/qmt_entry.c | 3 +++ + lustre/quota/qmt_lock.c | 46 +++++++++++++++++++++++++++------------- + lustre/quota/qmt_pool.c | 12 +++++++---- + 3 files changed, 42 insertions(+), 19 deletions(-) + +diff --git a/lustre/quota/qmt_entry.c b/lustre/quota/qmt_entry.c +index a37bf48e6c..182d33ada0 100644 +--- a/lustre/quota/qmt_entry.c ++++ b/lustre/quota/qmt_entry.c +@@ -1087,6 +1087,9 @@ struct lqe_glbl_data *qmt_alloc_lqe_gd(struct qmt_pool_info *pool, int qtype) + + void qmt_free_lqe_gd(struct lqe_glbl_data *lgd) + { ++ if (unlikely(!lgd)) ++ return; ++ + OBD_FREE(lgd->lqeg_arr, + sizeof(struct lqe_glbl_entry) * lgd->lqeg_num_alloc); + OBD_FREE(lgd, sizeof(struct lqe_glbl_data)); +diff --git a/lustre/quota/qmt_lock.c b/lustre/quota/qmt_lock.c +index 49421e3805..3592c41e37 100644 +--- a/lustre/quota/qmt_lock.c ++++ b/lustre/quota/qmt_lock.c +@@ -401,10 +401,13 @@ int qmt_lvbo_update(struct lu_device *ld, struct ldlm_resource *res, + if (rc) + GOTO(out_exp, rc); + +- if (need_revoke && qmt_set_revoke(env, lqe, rc, idx) && +- lqe->lqe_glbl_data) { +- qmt_seed_glbe_edquot(env, lqe->lqe_glbl_data); +- qmt_id_lock_notify(qmt, lqe); ++ if (need_revoke && qmt_set_revoke(env, lqe, rc, idx)) { ++ mutex_lock(&lqe->lqe_glbl_data_lock); ++ if (lqe->lqe_glbl_data) { ++ qmt_seed_glbe_edquot(env, lqe->lqe_glbl_data); ++ qmt_id_lock_notify(qmt, lqe); ++ } ++ mutex_unlock(&lqe->lqe_glbl_data_lock); + } + + if (lvb->lvb_id_rel) { +@@ -518,12 +521,13 @@ int qmt_lvbo_free(struct lu_device *ld, struct ldlm_resource *res) + + if (res->lr_name.name[LUSTRE_RES_ID_QUOTA_SEQ_OFF] != 0) { + struct lquota_entry *lqe = res->lr_lvb_data; +- struct lqe_glbl_data *lgd = lqe->lqe_glbl_data; ++ struct lqe_glbl_data *lgd; + + mutex_lock(&lqe->lqe_glbl_data_lock); ++ lgd = lqe->lqe_glbl_data; + lqe->lqe_glbl_data = NULL; +- qmt_free_lqe_gd(lgd); + mutex_unlock(&lqe->lqe_glbl_data_lock); ++ qmt_free_lqe_gd(lgd); + + /* release lqe reference */ + lqe_putref(lqe); +@@ -571,6 +575,7 @@ static int qmt_alloc_lock_array(struct ldlm_resource *res, + struct qmt_gl_lock_array *array, + qmt_glimpse_cb_t cb, void *arg) + { ++ struct lquota_entry *lqe = arg; + struct list_head *pos; + unsigned long count = 0; + int fail_cnt = 0; +@@ -578,6 +583,8 @@ static int qmt_alloc_lock_array(struct ldlm_resource *res, + + LASSERT(!array->q_max && !array->q_cnt && !array->q_locks); + again: ++ if (cb) ++ mutex_lock(&lqe->lqe_glbl_data_lock); + lock_res(res); + /* scan list of granted locks */ + list_for_each(pos, &res->lr_granted) { +@@ -601,6 +608,8 @@ again: + } + } + unlock_res(res); ++ if (cb) ++ mutex_unlock(&lqe->lqe_glbl_data_lock); + + if (count > array->q_max) { + qmt_free_lock_array(array); +@@ -628,7 +637,6 @@ void qmt_setup_id_desc(struct ldlm_lock *lock, union ldlm_gl_desc *desc, + struct lquota_entry *lqe) + { + struct obd_uuid *uuid = &(lock)->l_export->exp_client_uuid; +- struct lqe_glbl_data *lgd = lqe->lqe_glbl_data; + int idx, stype; + __u64 qunit; + bool edquot; +@@ -641,8 +649,18 @@ void qmt_setup_id_desc(struct ldlm_lock *lock, union ldlm_gl_desc *desc, + edquot = lqe->lqe_edquot; + qunit = lqe->lqe_qunit; + } else { +- edquot = lgd->lqeg_arr[idx].lge_edquot; +- qunit = lgd->lqeg_arr[idx].lge_qunit; ++ struct lqe_glbl_data *lgd; ++ ++ mutex_lock(&lqe->lqe_glbl_data_lock); ++ lgd = lqe->lqe_glbl_data; ++ if (lgd) { ++ edquot = lgd->lqeg_arr[idx].lge_edquot; ++ qunit = lgd->lqeg_arr[idx].lge_qunit; ++ } else { ++ edquot = lqe->lqe_edquot; ++ qunit = lqe->lqe_qunit; ++ } ++ mutex_unlock(&lqe->lqe_glbl_data_lock); + } + + /* fill glimpse descriptor with lqe settings */ +@@ -671,7 +689,6 @@ static int qmt_glimpse_lock(const struct lu_env *env, struct qmt_device *qmt, + qmt_glimpse_cb_t cb, struct lquota_entry *lqe) + { + union ldlm_gl_desc *descs = NULL; +- struct lqe_glbl_data *gld; + struct list_head *tmp, *pos; + LIST_HEAD(gl_list); + struct qmt_gl_lock_array locks; +@@ -679,7 +696,6 @@ static int qmt_glimpse_lock(const struct lu_env *env, struct qmt_device *qmt, + int rc = 0; + ENTRY; + +- gld = lqe ? lqe->lqe_glbl_data : NULL; + memset(&locks, 0, sizeof(locks)); + rc = qmt_alloc_lock_array(res, &locks, cb, lqe); + if (rc) { +@@ -696,7 +712,7 @@ static int qmt_glimpse_lock(const struct lu_env *env, struct qmt_device *qmt, + locks_count = locks.q_cnt; + + /* Use one desc for all works, when called from qmt_glb_lock_notify */ +- if (gld && locks.q_cnt > 1) { ++ if (cb && locks.q_cnt > 1) { + /* TODO: think about to store this preallocated descs + * in lqe_global in lqeg_arr as a part of lqe_glbl_entry. + * The benefit is that we don't need to allocate/free +@@ -723,7 +739,7 @@ static int qmt_glimpse_lock(const struct lu_env *env, struct qmt_device *qmt, + continue; + } + +- if (gld) { ++ if (cb) { + if (descs) + desc = &descs[i - 1]; + qmt_setup_id_desc(locks.q_locks[i - 1], desc, lqe); +@@ -842,8 +858,8 @@ static int qmt_id_lock_cb(struct ldlm_lock *lock, struct lquota_entry *lqe) + if (lqe_rtype(lqe) == LQUOTA_RES_DT && stype == QMT_STYPE_MDT) + return 1; + else +- return lgd->lqeg_arr[idx].lge_edquot_nu || +- lgd->lqeg_arr[idx].lge_qunit_nu; ++ return lgd ? lgd->lqeg_arr[idx].lge_edquot_nu || ++ lgd->lqeg_arr[idx].lge_qunit_nu : 0; + } + + +diff --git a/lustre/quota/qmt_pool.c b/lustre/quota/qmt_pool.c +index c4398a5bbd..d9dc4a165c 100644 +--- a/lustre/quota/qmt_pool.c ++++ b/lustre/quota/qmt_pool.c +@@ -1092,10 +1092,14 @@ static int qmt_site_recalc_cb(struct cfs_hash *hs, struct cfs_hash_bd *bd, + /* Find all lqes with lqe_id to reseed lgd array */ + rc = qmt_pool_lqes_lookup_spec(env, qmt, lqe_rtype(lqe), + lqe_qtype(lqe), &lqe->lqe_id); +- if (!rc && qti_lqes_glbl(env)->lqe_glbl_data) { +- qmt_seed_glbe(env, +- qti_lqes_glbl(env)->lqe_glbl_data); +- qmt_id_lock_notify(qmt, qti_lqes_glbl(env)); ++ if (!rc) { ++ struct lquota_entry *lqeg = qti_lqes_glbl(env); ++ ++ mutex_lock(&lqeg->lqe_glbl_data_lock); ++ if (lqeg->lqe_glbl_data) ++ qmt_seed_glbe(env, lqeg->lqe_glbl_data); ++ mutex_unlock(&lqeg->lqe_glbl_data_lock); ++ qmt_id_lock_notify(qmt, lqeg); + } + qti_lqes_fini(env); + } +-- +2.33.0 + diff --git a/0049-LU-13135-quota-improve-checks-in-OSDs-to-ignore-quot.patch b/0010-LU-13135-quota-improve-checks-in-OSDs-to-ignore-quot.patch similarity index 90% rename from 0049-LU-13135-quota-improve-checks-in-OSDs-to-ignore-quot.patch rename to 0010-LU-13135-quota-improve-checks-in-OSDs-to-ignore-quot.patch index 39711236229e29225c36d1d975d9ba9753a8f712..3ed9493d675cdc2077ccd2262b4d052e2d4308e3 100644 --- a/0049-LU-13135-quota-improve-checks-in-OSDs-to-ignore-quot.patch +++ b/0010-LU-13135-quota-improve-checks-in-OSDs-to-ignore-quot.patch @@ -1,7 +1,7 @@ -From 7495ac45845283a6abf5cb12baf42cb876159081 Mon Sep 17 00:00:00 2001 +From 407edcfc3159e9bfdfdde8f06af74008b03f81f0 Mon Sep 17 00:00:00 2001 From: Alex Zhuravlev Date: Tue, 14 Jan 2020 22:38:51 +0300 -Subject: [PATCH 49/61] LU-13135 quota: improve checks in OSDs to ignore quota +Subject: [PATCH 10/49] LU-13135 quota: improve checks in OSDs to ignore quota for root-owned files. @@ -13,15 +13,21 @@ ave.write declaration in sanity/60a: zfs before 3.21 usec, after 1.16 usec ldiskfs before 4.06 usec, after 0.66 usec +Note that this patch is related to bug LU-15726. +Lustre-change: https://review.whamcloud.com/37232 +Lustre-commit: ef90a02d126312b0d2f28b8c6eb2d299c68eb936 + Change-Id: Ib9ba50d260eac408f1f5e43c4d722ff5024135cf Signed-off-by: Alex Zhuravlev -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/37232 -Reviewed-by: Oleg Drokin Reviewed-by: Andreas Dilger Reviewed-by: Wang Shilong -Tested-by: jenkins -Tested-by: Maloo Signed-off-by: Xinliang Liu +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51776 +Tested-by: Maloo +Tested-by: jenkins +Reviewed-by: Shaun Tancheff +Reviewed-by: James Simmons +Reviewed-by: Oleg Drokin --- lustre/osd-ldiskfs/osd_handler.c | 6 ++++++ lustre/osd-ldiskfs/osd_quota.c | 4 ++++ diff --git a/0010-LU-16717-mdt-treat-unknown-hash-type-as-sane-type.patch b/0010-LU-16717-mdt-treat-unknown-hash-type-as-sane-type.patch deleted file mode 100644 index f20d906bb8d0bbf3be285c44e928f82c29259007..0000000000000000000000000000000000000000 --- a/0010-LU-16717-mdt-treat-unknown-hash-type-as-sane-type.patch +++ /dev/null @@ -1,155 +0,0 @@ -From e4208468b65a34c84c20d5d932f35b29f9025722 Mon Sep 17 00:00:00 2001 -From: Lai Siyao -Date: Sun, 23 Apr 2023 04:09:02 -0400 -Subject: [PATCH 10/61] LU-16717 mdt: treat unknown hash type as sane type - -Directory migration failure may leave directory hash type as -LMV_HASH_TYPE_UNKNOWN|LMV_HASH_FLAG_BAD_TYPE, which should be treated -as sane hash type on existing directories, otherwise such directories -can't be unlinked. - -Add sanity 230y. - -Lustre-change: https://review.whamcloud.com/50796 -Lustre-commit: 05cdb71ba6813570123613993f3cfcf74fc83561 - -Signed-off-by: Lai Siyao -Change-Id: Ieffc0808d1db989d0bf9723f05cddb06f349e208 -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/50796 -Reviewed-by: Andreas Dilger -Reviewed-by: Hongchao Zhang -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51235 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/include/lustre_lmv.h | 4 +-- - lustre/include/obd_support.h | 1 + - .../include/uapi/linux/lustre/lustre_user.h | 10 ++++++++ - lustre/mdt/mdt_reint.c | 18 +++++++++++++ - lustre/tests/sanity.sh | 25 +++++++++++++++++++ - 5 files changed, 56 insertions(+), 2 deletions(-) - -diff --git a/lustre/include/lustre_lmv.h b/lustre/include/lustre_lmv.h -index 2ffd77fc57..b848408da9 100644 ---- a/lustre/include/lustre_lmv.h -+++ b/lustre/include/lustre_lmv.h -@@ -438,7 +438,7 @@ static inline bool lmv_is_sane(const struct lmv_mds_md_v1 *lmv) - if (le32_to_cpu(lmv->lmv_stripe_count) == 0) - goto insane; - -- if (!lmv_is_known_hash_type(le32_to_cpu(lmv->lmv_hash_type))) -+ if (!lmv_is_sane_hash_type(le32_to_cpu(lmv->lmv_hash_type))) - goto insane; - - return true; -@@ -460,7 +460,7 @@ static inline bool lmv_is_sane2(const struct lmv_mds_md_v1 *lmv) - if (le32_to_cpu(lmv->lmv_stripe_count) == 0) - goto insane; - -- if (!lmv_is_known_hash_type(le32_to_cpu(lmv->lmv_hash_type))) -+ if (!lmv_is_sane_hash_type(le32_to_cpu(lmv->lmv_hash_type))) - goto insane; - - return true; -diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h -index cca40a688b..5a6998043b 100644 ---- a/lustre/include/obd_support.h -+++ b/lustre/include/obd_support.h -@@ -681,6 +681,7 @@ extern char obd_jobid_var[]; - - /* MIGRATE */ - #define OBD_FAIL_MIGRATE_ENTRIES 0x1801 -+#define OBD_FAIL_MIGRATE_BAD_HASH 0x1802 - - /* LMV */ - #define OBD_FAIL_UNKNOWN_LMV_STRIPE 0x1901 -diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h -index b2ae5992d2..25fa6ad28d 100644 ---- a/lustre/include/uapi/linux/lustre/lustre_user.h -+++ b/lustre/include/uapi/linux/lustre/lustre_user.h -@@ -1046,6 +1046,16 @@ static inline bool lmv_is_known_hash_type(__u32 type) - - #define LMV_HASH_FLAG_KNOWN 0xbe000000 - -+/* migration failure may leave hash type as -+ * LMV_HASH_TYPE_UNKNOWN|LMV_HASH_FLAG_BAD_TYPE, which should be treated as -+ * sane, so such directory can be accessed (resume migration or unlink). -+ */ -+static inline bool lmv_is_sane_hash_type(__u32 type) -+{ -+ return lmv_is_known_hash_type(type) || -+ type == (LMV_HASH_TYPE_UNKNOWN | LMV_HASH_FLAG_BAD_TYPE); -+} -+ - /* both SPLIT and MIGRATION are set for directory split */ - static inline bool lmv_hash_is_splitting(__u32 hash) - { -diff --git a/lustre/mdt/mdt_reint.c b/lustre/mdt/mdt_reint.c -index c800ca64be..a2ddbbcf07 100644 ---- a/lustre/mdt/mdt_reint.c -+++ b/lustre/mdt/mdt_reint.c -@@ -2312,6 +2312,24 @@ int mdt_reint_migrate(struct mdt_thread_info *info, - if (rc) - GOTO(put_parent, rc); - -+ if (CFS_FAIL_CHECK(OBD_FAIL_MIGRATE_BAD_HASH) && -+ (ma->ma_valid & MA_LMV) && -+ lmv_is_migrating(&ma->ma_lmv->lmv_md_v1)) { -+ struct lu_buf *buf = &info->mti_buf; -+ struct lmv_mds_md_v1 *lmv = &ma->ma_lmv->lmv_md_v1; -+ __u32 version = le32_to_cpu(lmv->lmv_layout_version); -+ -+ lmv->lmv_hash_type = cpu_to_le32(LMV_HASH_TYPE_UNKNOWN | -+ LMV_HASH_FLAG_BAD_TYPE); -+ lmv->lmv_layout_version = cpu_to_le32(version + 1); -+ buf->lb_buf = lmv; -+ buf->lb_len = sizeof(*lmv); -+ rc = mo_xattr_set(env, mdt_object_child(pobj), buf, -+ XATTR_NAME_LMV, LU_XATTR_REPLACE); -+ mo_invalidate(env, mdt_object_child(pobj)); -+ GOTO(put_parent, rc); -+ } -+ - lock_parent: - /* lock parent object */ - lhp = &info->mti_lh[MDT_LH_PARENT]; -diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh -index d73b285eca..6cd539a737 100755 ---- a/lustre/tests/sanity.sh -+++ b/lustre/tests/sanity.sh -@@ -20636,6 +20636,31 @@ test_230w() { - } - run_test 230w "non-recursive mode dir migration" - -+test_230y() { -+ (( MDSCOUNT > 1 )) || skip "needs >= 2 MDTs" -+ (( MDS1_VERSION >= $(version_code 2.15.3) )) || -+ skip "Need MDS version at least 2.15.3" -+ -+ local pid -+ -+ test_mkdir -c -1 $DIR/$tdir || error "mkdir $tdir failed" -+ $LFS getdirstripe $DIR/$tdir -+ createmany -d $DIR/$tdir/d 100 || error "createmany failed" -+ $LFS migrate -m 1 -c 2 $DIR/$tdir & -+ pid=$! -+ sleep 1 -+ -+ #OBD_FAIL_MIGRATE_BAD_HASH 0x1802 -+ do_facet mds2 lctl set_param fail_loc=0x1802 -+ -+ wait $pid -+ do_facet mds2 lctl set_param fail_loc=0 -+ $LFS getdirstripe $DIR/$tdir -+ unlinkmany -d $DIR/$tdir/d 100 || error "unlinkmany failed" -+ rmdir $DIR/$tdir || error "rmdir $tdir failed" -+} -+run_test 230y "unlink dir with bad hash type" -+ - test_231a() - { - # For simplicity this test assumes that max_pages_per_rpc --- -2.33.0 - diff --git a/0011-LU-15481-llog-Add-LLOG_SKIP_PLAIN-to-skip-llog-plain.patch b/0011-LU-15481-llog-Add-LLOG_SKIP_PLAIN-to-skip-llog-plain.patch deleted file mode 100644 index 45ceec2df09047888d3dbd0f0e93e21526e3afeb..0000000000000000000000000000000000000000 --- a/0011-LU-15481-llog-Add-LLOG_SKIP_PLAIN-to-skip-llog-plain.patch +++ /dev/null @@ -1,170 +0,0 @@ -From 053ab4a47a3391ff2b3b252b649b88e4579ab88f Mon Sep 17 00:00:00 2001 -From: Etienne AUJAMES -Date: Wed, 22 Feb 2023 11:18:49 -0800 -Subject: [PATCH 11/61] LU-15481 llog: Add LLOG_SKIP_PLAIN to skip llog plain - -Add the catalog callback return LLOG_SKIP_PLAIN to conditionally skip -an entire llog plain. - -This could speedup the catalog processing for specific usages when a -record need to be access in the "middle" of the catalog. This could -be usefull for changelog with several users or HSM. - -This patch modify chlg_read_cat_process_cb() to use LLOG_SKIP_PLAIN. -The main idea came from: d813c75d ("LU-14688 mdt: changelog purge -deletes plain llog") - -**Performance test:** - -* Environement: -2474195 changelogs record store on the mds0 (40 llog plain): -mds# lctl get_param -n mdd.lustrefs-MDT0000.changelog_users -current index: 2474195 -ID index (idle seconds) -cl1 0 (3509) - -* Test -Access to records at the end of the catalog (offset: 2474194): -client# time lfs changelog lustrefs-MDT0000 2474194 >/dev/null - -* Results -- with the patch: real 0m0.592s -- without the patch: real 0m17.835s (x30) - -Lustre-change: https://review.whamcloud.com/46310 -Lustre-commit: aa22a6826ee521ab14994a4533b0dbffb529aab0 - -Signed-off-by: Etienne AUJAMES -Change-Id: I887d5bef1f3a6a31c46bc58959e0f508266c53d2 -Reviewed-by: Alexander Boyko -Reviewed-by: Andreas Dilger -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/48771 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin -Reviewed-by: Etienne AUJAMES ---- - lustre/include/lustre_log.h | 17 +++++++++++++++++ - lustre/mdc/mdc_changelog.c | 5 +++++ - lustre/mdd/mdd_device.c | 4 ++-- - lustre/obdclass/llog.c | 6 ++++-- - lustre/obdclass/llog_cat.c | 6 ++++++ - 5 files changed, 34 insertions(+), 4 deletions(-) - -diff --git a/lustre/include/lustre_log.h b/lustre/include/lustre_log.h -index 360ba26dd5..1fdca73715 100644 ---- a/lustre/include/lustre_log.h -+++ b/lustre/include/lustre_log.h -@@ -340,6 +340,7 @@ struct llog_ctxt { - #define LLOG_PROC_BREAK 0x0001 - #define LLOG_DEL_RECORD 0x0002 - #define LLOG_DEL_PLAIN 0x0003 -+#define LLOG_SKIP_PLAIN 0x0004 - - static inline int llog_obd2ops(struct llog_ctxt *ctxt, - const struct llog_operations **lop) -@@ -519,6 +520,22 @@ static inline int llog_is_full(struct llog_handle *llh) - return llh->lgh_last_idx >= LLOG_HDR_BITMAP_SIZE(llh->lgh_hdr) - 1; - } - -+/* Determine if a llog plain of a catalog could be skiped based on record -+ * custom indexes. -+ * This assumes that indexes follow each other. The number of records to skip -+ * can be computed base on a starting offset and the index of the current -+ * record (in llog catalog callback). -+ */ -+static inline int llog_is_plain_skipable(struct llog_log_hdr *lh, -+ struct llog_rec_hdr *rec, -+ __u64 curr, __u64 start) -+{ -+ if (start == 0 || curr >= start) -+ return 0; -+ -+ return (LLOG_HDR_BITMAP_SIZE(lh) - rec->lrh_index) < (start - curr); -+} -+ - struct llog_cfg_rec { - struct llog_rec_hdr lcr_hdr; - struct lustre_cfg lcr_cfg; -diff --git a/lustre/mdc/mdc_changelog.c b/lustre/mdc/mdc_changelog.c -index 843c4de8a4..aea492b9f5 100644 ---- a/lustre/mdc/mdc_changelog.c -+++ b/lustre/mdc/mdc_changelog.c -@@ -228,6 +228,11 @@ static int chlg_read_cat_process_cb(const struct lu_env *env, - RETURN(rc); - } - -+ /* Check if we can skip the entire llog plain */ -+ if (llog_is_plain_skipable(llh->lgh_hdr, hdr, rec->cr.cr_index, -+ crs->crs_start_offset)) -+ RETURN(LLOG_SKIP_PLAIN); -+ - /* Skip undesired records */ - if (rec->cr.cr_index < crs->crs_start_offset) - RETURN(0); -diff --git a/lustre/mdd/mdd_device.c b/lustre/mdd/mdd_device.c -index 37667afde0..94fb1f64a5 100644 ---- a/lustre/mdd/mdd_device.c -+++ b/lustre/mdd/mdd_device.c -@@ -393,8 +393,8 @@ static int llog_changelog_cancel_cb(const struct lu_env *env, - * last cr_index at this plain llog. And if it less then cookie endrec - * cancel the whole file. - */ -- if ((LLOG_HDR_BITMAP_SIZE(llh->lgh_hdr) - hdr->lrh_index + -- rec->cr.cr_index) < cl_cookie->endrec) { -+ if (llog_is_plain_skipable(llh->lgh_hdr, hdr, rec->cr.cr_index, -+ cl_cookie->endrec)) { - int rc; - - if (unlikely(OBD_FAIL_PRECHECK(OBD_FAIL_MDS_CHANGELOG_DEL))) { -diff --git a/lustre/obdclass/llog.c b/lustre/obdclass/llog.c -index 2c45c9673a..79f89fe365 100644 ---- a/lustre/obdclass/llog.c -+++ b/lustre/obdclass/llog.c -@@ -736,7 +736,8 @@ repeat: - lgc->lgc_index = tmp_idx; - } - -- if (rc == LLOG_PROC_BREAK) { -+ if (rc == LLOG_PROC_BREAK || -+ rc == LLOG_SKIP_PLAIN) { - GOTO(out, rc); - } else if (rc == LLOG_DEL_RECORD) { - rc = llog_cancel_rec(lpi->lpi_env, -@@ -1005,7 +1006,8 @@ int llog_reverse_process(const struct lu_env *env, - sizeof(*tail); - - rc = cb(env, loghandle, rec, data); -- if (rc == LLOG_PROC_BREAK) { -+ if (rc == LLOG_PROC_BREAK || -+ rc == LLOG_SKIP_PLAIN) { - GOTO(out, rc); - } else if (rc == LLOG_DEL_RECORD) { - rc = llog_cancel_rec(env, loghandle, -diff --git a/lustre/obdclass/llog_cat.c b/lustre/obdclass/llog_cat.c -index ba44ad3003..4f5b0b1133 100644 ---- a/lustre/obdclass/llog_cat.c -+++ b/lustre/obdclass/llog_cat.c -@@ -895,6 +895,9 @@ out: - if (rc == LLOG_DEL_PLAIN || rc == LLOG_DEL_RECORD) - /* clear wrong catalog entry */ - rc = llog_cat_cleanup(env, cat_llh, llh, rec->lrh_index); -+ else if (rc == LLOG_SKIP_PLAIN) -+ /* processing callback ask to skip the llog -> continue */ -+ rc = 0; - - if (llh) - llog_handle_put(env, llh); -@@ -1060,6 +1063,9 @@ static int llog_cat_reverse_process_cb(const struct lu_env *env, - } else if (rc == LLOG_DEL_RECORD) { - /* clear wrong catalog entry */ - rc = llog_cat_cleanup(env, cat_llh, NULL, rec->lrh_index); -+ } else if (rc == LLOG_SKIP_PLAIN) { -+ /* processing callback ask to skip the llog -> continue */ -+ rc = 0; - } - if (rc) - RETURN(rc); --- -2.33.0 - diff --git a/0059-LU-16019-llite-fully-disable-readahead-in-kernel-I-O.patch b/0011-LU-16019-llite-fully-disable-readahead-in-kernel-I-O.patch similarity index 84% rename from 0059-LU-16019-llite-fully-disable-readahead-in-kernel-I-O.patch rename to 0011-LU-16019-llite-fully-disable-readahead-in-kernel-I-O.patch index 2a429ac8f5092dc5d1ba3c12bba1bc119d1250f8..9c5fda46f9d2fe7c92173383823959f7f01e3a7a 100644 --- a/0059-LU-16019-llite-fully-disable-readahead-in-kernel-I-O.patch +++ b/0011-LU-16019-llite-fully-disable-readahead-in-kernel-I-O.patch @@ -1,7 +1,7 @@ -From 4a3d324f5827d64216c4b7ea62adbfb5183bb795 Mon Sep 17 00:00:00 2001 +From 508d74f0a5b5c58cbfe51a6e9a10327892f7f0f3 Mon Sep 17 00:00:00 2001 From: Qian Yingjin Date: Mon, 15 Aug 2022 11:15:25 -0700 -Subject: [PATCH 59/61] LU-16019 llite: fully disable readahead in kernel I/O +Subject: [PATCH 11/49] LU-16019 llite: fully disable readahead in kernel I/O path In the new kernel (rhel9 or ubuntu 2204), the readahead path may @@ -62,18 +62,22 @@ Signed-off-by: Qian Yingjin Change-Id: I859a6404abb9116d9acfa03de91e61d3536d3554 Reviewed-by: Andreas Dilger Reviewed-by: Li Xi -Signed-off-by: Xinliang Liu +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/48219 +Reviewed-by: Oleg Drokin +Reviewed-by: xinliang +Tested-by: jenkins +Tested-by: Maloo --- lustre/autoconf/lustre-core.m4 | 22 ++++++++++++++++++++++ lustre/llite/llite_lib.c | 3 +++ 2 files changed, 25 insertions(+) diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 -index 251c977ea4..1bfc917dde 100644 +index 7713b8cf45..064347b8c0 100644 --- a/lustre/autoconf/lustre-core.m4 +++ b/lustre/autoconf/lustre-core.m4 -@@ -1964,6 +1964,27 @@ posix_acl_update_mode, [ - ]) +@@ -2161,6 +2161,27 @@ AC_DEFUN([LC_POSIX_ACL_UPDATE_MODE], [ + ]) ]) # LC_POSIX_ACL_UPDATE_MODE +# @@ -100,7 +104,7 @@ index 251c977ea4..1bfc917dde 100644 # # LC_IOP_GENERIC_READLINK # -@@ -2990,6 +3011,7 @@ AC_DEFUN([LC_PROG_LINUX], [ +@@ -3790,6 +3811,7 @@ AC_DEFUN([LC_PROG_LINUX_RESULTS], [ LC_GROUP_INFO_GID LC_VFS_SETXATTR LC_POSIX_ACL_UPDATE_MODE @@ -109,7 +113,7 @@ index 251c977ea4..1bfc917dde 100644 # 4.10 LC_IOP_GENERIC_READLINK diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c -index f67ea3f7eb..ab8363e7a3 100644 +index 5121d456c0..6addf6caf1 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -1366,6 +1366,9 @@ int ll_fill_super(struct super_block *sb) diff --git a/0012-LU-15655-contrib-update-branch_comm-to-python3.patch b/0012-LU-15655-contrib-update-branch_comm-to-python3.patch new file mode 100644 index 0000000000000000000000000000000000000000..1f37337a8b859ddd224dccc6ded9cc788cccd576 --- /dev/null +++ b/0012-LU-15655-contrib-update-branch_comm-to-python3.patch @@ -0,0 +1,127 @@ +From 32a1d7a86150621391d7501d1a5ed7b8a25d80c3 Mon Sep 17 00:00:00 2001 +From: "John L. Hammond" +Date: Thu, 17 Mar 2022 08:55:32 -0500 +Subject: [PATCH 12/49] LU-15655 contrib: update branch_comm to python3 + +Update the branch_comm script from to python2 to python3. + +Lustre-change: https://review.whamcloud.com/46856 +Lustre-commit: c6b257254878387976247fea29bd2b142cd03e5e + +Test-Parameters: trivial +Signed-off-by: John L. Hammond +Change-Id: Icbb747c01e15cbe77c6dd6df3e913654ded84239 +Reviewed-by: Jian Yu +Reviewed-by: Andreas Dilger +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53947 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + contrib/scripts/branch_comm | 25 +++++++++++++------------ + 1 file changed, 13 insertions(+), 12 deletions(-) + +diff --git a/contrib/scripts/branch_comm b/contrib/scripts/branch_comm +index dd12725df6..badfa61d38 100755 +--- a/contrib/scripts/branch_comm ++++ b/contrib/scripts/branch_comm +@@ -1,4 +1,4 @@ +-#!/usr/bin/env python2 ++#!/usr/bin/env python3 + + import re + import subprocess +@@ -47,8 +47,8 @@ GIT_LOG_FORMAT = '%x1f'.join(GIT_LOG_KEYS) + '%x1e' + + def _change_from_record(rec): + change = Change() +- change.__dict__.update(dict(zip(GIT_LOG_FIELDS, rec.split('\x1f')))) +- change.author_date = long(change.author_date) ++ change.__dict__.update(dict(list(zip(GIT_LOG_FIELDS, rec.split('\x1f'))))) ++ change.author_date = int(change.author_date) + for line in change.body.splitlines(): + # Sometimes we have 'key : value' so we strip both sides. + lis = line.split(':', 1) +@@ -140,7 +140,8 @@ class Branch(object): + '--reverse', + self.name + ] + self.paths, +- stdout=subprocess.PIPE) ++ stdout=subprocess.PIPE, ++ text=True) + out, _ = pipe.communicate() + rc = pipe.wait() + assert rc == 0 +@@ -234,7 +235,7 @@ def branch_comm(b1, b2): + i1 += 1 + i2 += 1 + # c1 is common to both branches. +- print '\t\t%s\t%s' % (c1.commit, c1.subject) # TODO Add a '*' if subjects different... ++ print('\t\t%s\t%s' % (c1.commit, c1.subject)) # TODO Add a '*' if subjects different... + continue + + if p1 and not p2: +@@ -242,7 +243,7 @@ def branch_comm(b1, b2): + change_set_printed(c1) + i1 += 1 + # c1 is unique to b1. +- print '%s\t\t\t%s' % (c1.commit, c1.subject) ++ print('%s\t\t\t%s' % (c1.commit, c1.subject)) + continue + + if p2 and not p1: +@@ -250,7 +251,7 @@ def branch_comm(b1, b2): + change_set_printed(c2) + i2 += 1 + # c2 is unique to b2. +- print '\t%s\t\t%s' % (c2.commit, c2.subject) ++ print('\t%s\t\t%s' % (c2.commit, c2.subject)) + continue + + # Now neither is ported or both are ported (and the order is weird). +@@ -259,13 +260,13 @@ def branch_comm(b1, b2): + change_set_printed(p2) + i1 += 1 + # c1 is common to both branches. +- print '\t\t%s\t%s' % (c1.commit, c1.subject) ++ print('\t\t%s\t%s' % (c1.commit, c1.subject)) + continue + else: + change_set_printed(c1) + i1 += 1 + # c1 is unique to b1. +- print '%s\t\t\t%s' % (c1.commit, c1.subject) ++ print('%s\t\t\t%s' % (c1.commit, c1.subject)) + continue + + for c1 in b1.log[i1:]: +@@ -278,7 +279,7 @@ def branch_comm(b1, b2): + # c1 is unique to b1 and must be printed. + + change_set_printed(c1) +- print '%s\t\t\t%s' % (c1.commit, c1.subject) ++ print('%s\t\t\t%s' % (c1.commit, c1.subject)) + + for c2 in b2.log[i2:]: + if change_is_printed(c2): +@@ -287,7 +288,7 @@ def branch_comm(b1, b2): + assert i1 == n1 + # ... + change_set_printed(c2) +- print '\t%s\t\t%s' % (c2.commit, c2.subject) ++ print('\t%s\t\t%s' % (c2.commit, c2.subject)) + + + USAGE = """usage: '_PROGNAME_ BRANCH1 BRANCH2 [PATH]...' +@@ -320,7 +321,7 @@ repositories (for example 'origin/master' and 'other/b_post_cmd3') do: + + def main(): + if len(sys.argv) < 3: +- print >> sys.stderr, USAGE.replace('_PROGNAME_', sys.argv[0]) ++ print(USAGE.replace('_PROGNAME_', sys.argv[0]), file=sys.stderr) + sys.exit(1) + + paths = sys.argv[3:] +-- +2.33.0 + diff --git a/0012-LU-6612-utils-strengthen-llog_reader-vs-wrong-format.patch b/0012-LU-6612-utils-strengthen-llog_reader-vs-wrong-format.patch deleted file mode 100644 index 43c82ae57f9fa191007f9f4710a59496b91b4f1b..0000000000000000000000000000000000000000 --- a/0012-LU-6612-utils-strengthen-llog_reader-vs-wrong-format.patch +++ /dev/null @@ -1,74 +0,0 @@ -From badba63a54e905129dbdf28e31026580453ea337 Mon Sep 17 00:00:00 2001 -From: Bruno Faccini -Date: Wed, 22 Feb 2023 11:21:06 -0800 -Subject: [PATCH 12/61] LU-6612 utils: strengthen llog_reader vs wrong - format/header - -The following snippet shows that llog_reader can be puzzled due to -an invalid 0 for the number of records when parsing an expected -LLOG file header : -root# dd if=/dev/zero bs=4096 count=1 of=/tmp/zeroes -1+0 records in -1+0 records out -4096 bytes (4.1 kB) copied, 0.000263962 s, 15.5 MB/s -root# llog_reader /tmp/zeroes -Memory Alloc for recs_buf error. -Could not pack buffer; rc=-12 - -Lustre-change: https://review.whamcloud.com/15654 -Lustre-commit: 45291b8c06eebf33d3654db3a7d3cfc5836004a6 - -Test-Parameters: trivial testlist=sanity,sanity-hsm -Signed-off-by: Bruno Faccini -Change-Id: I12be79e6c6a5da384a5fd81878a76a7ea8aa5834 -Reviewed-by: Andreas Dilger -Reviewed-by: Mike Pershin -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/48900 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin -Reviewed-by: Mikhail Pershin ---- - lustre/utils/llog_reader.c | 8 ++++++-- - 1 file changed, 6 insertions(+), 2 deletions(-) - -diff --git a/lustre/utils/llog_reader.c b/lustre/utils/llog_reader.c -index dc2ef05127..153a471cc2 100644 ---- a/lustre/utils/llog_reader.c -+++ b/lustre/utils/llog_reader.c -@@ -222,6 +222,9 @@ int llog_pack_buffer(int fd, struct llog_log_hdr **llog, - int count; - int i, last_idx; - -+ *recs = NULL; -+ *recs_number = 0; -+ - rc = fstat(fd, &st); - if (rc < 0) { - rc = -errno; -@@ -270,11 +273,13 @@ int llog_pack_buffer(int fd, struct llog_log_hdr **llog, - } else if (count == 0) { - llapi_printf(LLAPI_MSG_NORMAL, - "uninitialized llog: zero record number\n"); -- *recs_number = 0; - goto clear_file_buf; - } -+ - /* the llog header not countable here.*/ - recs_num = count - 1; -+ if (recs_num == 0) -+ goto clear_file_buf; - - recs_buf = calloc(recs_num, sizeof(**recs_pr)); - if (!recs_buf) { -@@ -343,7 +348,6 @@ int llog_pack_buffer(int fd, struct llog_log_hdr **llog, - - *recs = recs_pr; - *recs_number = recs_num; -- - out: - return rc; - --- -2.33.0 - diff --git a/0013-LU-16052-llog-handle-EBADR-for-catalog-processing.patch b/0013-LU-16052-llog-handle-EBADR-for-catalog-processing.patch deleted file mode 100644 index 486e3472ac5d4f2b177e337eb2131d382b2190b9..0000000000000000000000000000000000000000 --- a/0013-LU-16052-llog-handle-EBADR-for-catalog-processing.patch +++ /dev/null @@ -1,46 +0,0 @@ -From b996d1e0276fdf6c084410cd1dcfac0df13437fe Mon Sep 17 00:00:00 2001 -From: Mikhail Pershin -Date: Mon, 17 Oct 2022 16:29:52 -0700 -Subject: [PATCH 13/61] LU-16052 llog: handle -EBADR for catalog processing - -Llog catalog processing might retry to get the last llog block -to check for new records if any. That might return -EBADR code -which should be considered as valid. Previously -EIO was -returned in all cases. - -Run conf-sanity test_106 several times as specific test - -Lustre-change: https://review.whamcloud.com/48070 -Lustre-commit: e260f751f2a21fa126eeb4bc9e94250ba3e815f1 - -Test-Parameters: testlist=conf-sanity env=ONLY=106,SLOW=yes,ONLY_REPEAT=10 -Signed-off-by: Mikhail Pershin -Change-Id: I30e04ba2c91c8bdce72c95675a1209639e9f0570 -Reviewed-by: Andreas Dilger -Reviewed-by: Etienne AUJAMES -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/48772 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/obdclass/llog.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/lustre/obdclass/llog.c b/lustre/obdclass/llog.c -index 79f89fe365..b23645d8b4 100644 ---- a/lustre/obdclass/llog.c -+++ b/lustre/obdclass/llog.c -@@ -576,8 +576,8 @@ repeat: - GOTO(out, rc = 0); - /* we`ve tried to reread the chunk, but there is no - * new records */ -- if (rc == -EIO && repeated && (chunk_offset + buf_offset) == -- cur_offset) -+ if (repeated && (chunk_offset + buf_offset) == cur_offset && -+ (rc == -EBADR || rc == -EIO)) - GOTO(out, rc = 0); - if (rc != 0) - GOTO(out, rc); --- -2.33.0 - diff --git a/0013-LU-16101-tests-skip-sanity-27J-for-more-kernels.patch b/0013-LU-16101-tests-skip-sanity-27J-for-more-kernels.patch new file mode 100644 index 0000000000000000000000000000000000000000..f0fc1397cb4f0b46602e6d6a86389db4c4cff213 --- /dev/null +++ b/0013-LU-16101-tests-skip-sanity-27J-for-more-kernels.patch @@ -0,0 +1,79 @@ +From da3b62d6937ce8095d0d46ae4b039cfba58b96d1 Mon Sep 17 00:00:00 2001 +From: Andreas Dilger +Date: Tue, 4 Jul 2023 17:08:03 -0600 +Subject: [PATCH 13/49] LU-16101 tests: skip sanity/27J for more kernels + +This is a bug in the kernel that is not present in older kernels +before commit v5.11-10234-gcbd59c48ae2b (5.12), and is fixed with +commit v6.2-rc4-61-g5956592ce337 (6.2). + +Move this from ALWAYS_EXCEPT (bug that needs to be fixed) to skip +(test that is known to fail in some configs but has been fixed). + +Lustre-change: https://review.whamcloud.com/51567 +Lustre-commit: b711af7d243f3773cec3a37f64c0e0aa8bbc363f + +Fixes: 63dd644 ("LU-16101 tests: add sanity/27J to always_except") +Test-Parameters: trivial testlist=sanity clientdistro=el9.2 env=ONLY=27J +Signed-off-by: Andreas Dilger +Change-Id: I8ec0a6d25a90e05672b039cd6c2b2fbf8a3ebbe5 +Reviewed-by: Arshad Hussain +Reviewed-by: Alex Deiter +Signed-off-by: Etienne AUJAMES +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54036 +Reviewed-by: Oleg Drokin +Tested-by: jenkins +Tested-by: Maloo +--- + lustre/tests/sanity.sh | 19 +++++++------------ + 1 file changed, 7 insertions(+), 12 deletions(-) + +diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh +index 4f79e27e13..a6f75b2328 100755 +--- a/lustre/tests/sanity.sh ++++ b/lustre/tests/sanity.sh +@@ -75,13 +75,6 @@ if (( $LINUX_VERSION_CODE >= $(version_code 4.18.0) && + ALWAYS_EXCEPT+=" 411" + fi + +-# skip basic ops on file with foreign LOV tests on 5.16.0+ kernels +-# until the filemap_read() issue is fixed +-if (( $LINUX_VERSION_CODE >= $(version_code 5.16.0) )); then +- # bug number for skipped test: LU-16101 +- ALWAYS_EXCEPT="$ALWAYS_EXCEPT 27J" +-fi +- + #skip ACL tests on RHEL8 and SLES15 until tests changed to use other users + if (( $(egrep -cw "^bin|^daemon" /etc/passwd) < 2 )); then + # bug number: LU-15259 LU-15259 +@@ -128,10 +121,6 @@ if [ -r /etc/SuSE-release ] || [ -r /etc/SUSE-brand ]; then + [ $sles_version -lt $(version_code 12.0.0) ] && + # bug number for skipped test: LU-3703 + ALWAYS_EXCEPT="$ALWAYS_EXCEPT 234" +- +- [ $sles_version -ge $(version_code 15.4.0) ] && +- # bug number for skipped test: LU-16101 +- ALWAYS_EXCEPT="$ALWAYS_EXCEPT 27J" + elif [ -r /etc/os-release ]; then + if grep -qi ubuntu /etc/os-release; then + ubuntu_version=$(version_code $(sed -n -e 's/"//g' \ +@@ -2772,9 +2761,15 @@ test_27I() { + run_test 27I "check that root dir striping does not break parent dir one" + + test_27J() { +- [[ $MDS1_VERSION -le $(version_code 2.12.51) ]] && ++ (( $MDS1_VERSION > $(version_code 2.12.51) )) || + skip "Need MDS version newer than 2.12.51" + ++ # skip basic ops on file with foreign LOV tests on 5.12-6.2 kernels ++ # until the filemap_read() issue is fixed by v6.2-rc4-61-g5956592ce337 ++ (( $LINUX_VERSION_CODE < $(version_code 5.12.0) || ++ $LINUX_VERSION_CODE >= $(version_code 6.2.0) )) || ++ skip "Need kernel < 5.12.0 or >= 6.2.0 for filemap_read() fix" ++ + test_mkdir $DIR/$tdir + local uuid1=$(cat /proc/sys/kernel/random/uuid) + local uuid2=$(cat /proc/sys/kernel/random/uuid) +-- +2.33.0 + diff --git a/0014-LU-16717-mdt-resume-dir-migration-with-bad_type.patch b/0014-LU-16717-mdt-resume-dir-migration-with-bad_type.patch deleted file mode 100644 index e8d45dc8cd68f11ae95f36a62a7bcfdc50429f32..0000000000000000000000000000000000000000 --- a/0014-LU-16717-mdt-resume-dir-migration-with-bad_type.patch +++ /dev/null @@ -1,144 +0,0 @@ -From 1c882aebeaac4970c78a3616f1dd96d0920d133f Mon Sep 17 00:00:00 2001 -From: Lai Siyao -Date: Fri, 28 Apr 2023 05:22:03 -0400 -Subject: [PATCH 14/61] LU-16717 mdt: resume dir migration with bad_type - -LFSCK may set hash type to "none,bad_type" upon migration failure, -set it back to "fnv_1a_64,migrating,bad_type,fixed" to allow -migration resumption. fnv_1a_64 is set because it's the default hash -type, and now that we don't know the hash type in the original -migration command, just try with it. - -LFSCK just add "bad_type" flag on such directory, so that such -migration can always be resumed in the future. - -Add sanity 230z. - -Lustre-change: https://review.whamcloud.com/50797 -Lustre-commit: 151650e468ab423e831c30d635ea380e0434a122 - -Signed-off-by: Lai Siyao -Change-Id: I19606aefcb9115e6724843785aea89a1c380e23f -Reviewed-by: Andreas Dilger -Reviewed-by: Hongchao Zhang -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51243 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/lfsck/lfsck_striped_dir.c | 6 +++++- - lustre/mdt/mdt_reint.c | 32 ++++++++++++++++++++++++++++++++ - lustre/tests/sanity.sh | 30 ++++++++++++++++++++++++++++++ - 3 files changed, 67 insertions(+), 1 deletion(-) - -diff --git a/lustre/lfsck/lfsck_striped_dir.c b/lustre/lfsck/lfsck_striped_dir.c -index 63a875403f..2aa2c7d6e3 100644 ---- a/lustre/lfsck/lfsck_striped_dir.c -+++ b/lustre/lfsck/lfsck_striped_dir.c -@@ -1567,7 +1567,11 @@ int lfsck_namespace_repair_bad_name_hash(const struct lu_env *env, - GOTO(log, rc = 1); - - *lmv2 = llmv->ll_lmv; -- lmv2->lmv_hash_type = LMV_HASH_TYPE_UNKNOWN | LMV_HASH_FLAG_BAD_TYPE; -+ /* only set BAD_TYPE here, do not clear hash type or MIGRATION flag, -+ * so that user can resume dir migration if this is caused by dir -+ * migration failure. -+ */ -+ lmv2->lmv_hash_type |= LMV_HASH_FLAG_BAD_TYPE; - rc = lfsck_namespace_set_lmv_master(env, com, parent, lmv2, - lfsck_dto2fid(shard), - llmv->ll_lmv.lmv_master_mdt_index, -diff --git a/lustre/mdt/mdt_reint.c b/lustre/mdt/mdt_reint.c -index a2ddbbcf07..23948879cd 100644 ---- a/lustre/mdt/mdt_reint.c -+++ b/lustre/mdt/mdt_reint.c -@@ -2207,6 +2207,17 @@ close: - return rc ?: rc2; - } - -+/* LFSCK used to clear hash type and MIGRATION flag upon migration failure */ -+static inline bool lmv_is_failed_migration(const struct lmv_mds_md_v1 *lmv) -+{ -+ return le32_to_cpu(lmv->lmv_hash_type) == -+ (LMV_HASH_TYPE_UNKNOWN | LMV_HASH_FLAG_BAD_TYPE) && -+ lmv_is_known_hash_type(le32_to_cpu(lmv->lmv_migrate_hash)) && -+ le32_to_cpu(lmv->lmv_migrate_offset) > 0 && -+ le32_to_cpu(lmv->lmv_migrate_offset) < -+ le32_to_cpu(lmv->lmv_stripe_count); -+} -+ - /* - * migrate file in below steps: - * 1. lock parent and its stripes -@@ -2384,6 +2395,27 @@ lock_parent: - if ((ma->ma_valid & MA_LMV) && - lmv_is_restriping(&ma->ma_lmv->lmv_md_v1)) - GOTO(unlock_links, rc = -EBUSY); -+ else if (lmv_is_failed_migration(&ma->ma_lmv->lmv_md_v1)) { -+ struct lu_buf *buf = &info->mti_buf; -+ struct lmv_mds_md_v1 *lmv = &ma->ma_lmv->lmv_md_v1; -+ __u32 version = le32_to_cpu(lmv->lmv_layout_version); -+ -+ /* migration failed before, and LFSCK cleared hash type -+ * and flags, fake it to resume migration. -+ */ -+ lmv->lmv_hash_type = -+ cpu_to_le32(LMV_HASH_TYPE_FNV_1A_64 | -+ LMV_HASH_FLAG_MIGRATION | -+ LMV_HASH_FLAG_BAD_TYPE | -+ LMV_HASH_FLAG_FIXED); -+ lmv->lmv_layout_version = cpu_to_le32(version + 1); -+ buf->lb_buf = lmv; -+ buf->lb_len = sizeof(*lmv); -+ rc = mo_xattr_set(env, mdt_object_child(sobj), buf, -+ XATTR_NAME_LMV, LU_XATTR_REPLACE); -+ mo_invalidate(env, mdt_object_child(sobj)); -+ GOTO(unlock_links, rc = -EALREADY); -+ } - } - - /* if migration HSM is allowed */ -diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh -index 6cd539a737..93f8c3b30c 100755 ---- a/lustre/tests/sanity.sh -+++ b/lustre/tests/sanity.sh -@@ -20661,6 +20661,36 @@ test_230y() { - } - run_test 230y "unlink dir with bad hash type" - -+test_230z() { -+ (( MDSCOUNT > 1 )) || skip "needs >= 2 MDTs" -+ (( MDS1_VERSION >= $(version_code 2.15.3) )) || -+ skip "Need MDS version at least 2.15.3" -+ -+ local pid -+ -+ test_mkdir -c -1 $DIR/$tdir || error "mkdir $tdir failed" -+ $LFS getdirstripe $DIR/$tdir -+ createmany -d $DIR/$tdir/d 100 || error "createmany failed" -+ $LFS migrate -m 1 -c 2 -H fnv_1a_64 $DIR/$tdir & -+ pid=$! -+ sleep 1 -+ -+ #OBD_FAIL_MIGRATE_BAD_HASH 0x1802 -+ do_facet mds2 lctl set_param fail_loc=0x1802 -+ -+ wait $pid -+ do_facet mds2 lctl set_param fail_loc=0 -+ $LFS getdirstripe $DIR/$tdir -+ -+ # resume migration -+ $LFS migrate -m 1 -c 2 -H fnv_1a_64 $DIR/$tdir || -+ error "resume migration failed" -+ $LFS getdirstripe $DIR/$tdir -+ [ $($LFS getdirstripe -H $DIR/$tdir) == "fnv_1a_64,fixed" ] || -+ error "migration is not finished" -+} -+run_test 230z "resume dir migration with bad hash type" -+ - test_231a() - { - # For simplicity this test assumes that max_pages_per_rpc --- -2.33.0 - diff --git a/0014-LU-17175-tests-fix-sanity-krb5-test_5-in-interop-mod.patch b/0014-LU-17175-tests-fix-sanity-krb5-test_5-in-interop-mod.patch new file mode 100644 index 0000000000000000000000000000000000000000..22aeda647ced6ba00decb874eeae49d585cdc031 --- /dev/null +++ b/0014-LU-17175-tests-fix-sanity-krb5-test_5-in-interop-mod.patch @@ -0,0 +1,52 @@ +From 01918803768c865c47776d5e38ee6eda64e95797 Mon Sep 17 00:00:00 2001 +From: Sebastien Buisson +Date: Tue, 20 Feb 2024 10:58:40 +0100 +Subject: [PATCH 14/49] LU-17175 tests: fix sanity-krb5 test_5 in interop mode + +With newer servers, the lsvcgssd daemon is automatically restarted +when the server needs to process a gss request. So do not error out +if a file system access succeeds whereas the daemon was explicitly +killed. + +Test-Parameters: trivial +Signed-off-by: Sebastien Buisson +Change-Id: I52ec49ca584f07100699274fe3a52ea28d227c17 +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54103 +Reviewed-by: Andreas Dilger +Reviewed-by: Aurelien Degremont +Reviewed-by: Oleg Drokin +Tested-by: jenkins +Tested-by: Maloo +--- + lustre/tests/sanity-krb5.sh | 14 ++++++++++++-- + 1 file changed, 12 insertions(+), 2 deletions(-) + +diff --git a/lustre/tests/sanity-krb5.sh b/lustre/tests/sanity-krb5.sh +index 9725e9a7fe..234e0c7b14 100755 +--- a/lustre/tests/sanity-krb5.sh ++++ b/lustre/tests/sanity-krb5.sh +@@ -304,9 +304,19 @@ test_5() { + + # flush context, and touch + $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT" +- $RUNAS touch $file2 && error 'should fail without lsvcgssd' ++ $RUNAS touch $file2 ++ if [ $? -eq 0 ]; then ++ # with newer servers, daemon is restarted automatically ++ if (( CLIENT_VERSION == MDS1_VERSION )); then ++ error '$RUNAS touch $file2 should fail without lsvcgssd' ++ else ++ echo "$RUNAS touch $file2 succeeded" ++ fi ++ else ++ echo "$RUNAS touch $file2 failed as expected" ++ fi + +- # restart lsvcgssd, expect touch suceed ++ # restart lsvcgssd, expect touch succeed + echo "restart lsvcgssd and recovering" + start_gss_daemons $(comma_list $(mdts_nodes)) "$LSVCGSSD -v" + sleep 5 +-- +2.33.0 + diff --git a/0015-LU-14668-lnet-Lock-primary-NID-logic.patch b/0015-LU-14668-lnet-Lock-primary-NID-logic.patch deleted file mode 100644 index 7a2a69d26b47a75d48c104a2abe7793af06c779a..0000000000000000000000000000000000000000 --- a/0015-LU-14668-lnet-Lock-primary-NID-logic.patch +++ /dev/null @@ -1,241 +0,0 @@ -From b341288179d9b3ad594b461586d826d6811db5a1 Mon Sep 17 00:00:00 2001 -From: Amir Shehata -Date: Wed, 5 May 2021 11:35:06 -0700 -Subject: [PATCH 15/61] LU-14668 lnet: Lock primary NID logic - -If a peer is created by Lustre make sure to lock that peer's -primary NID. This peer can be discovered in the background. -There is no need to block until discovery is complete, as Lustre -can continue on with the primary NID it provided. - -Discovery will populate the peer with other interfaces the peer has -but will not change the peer's primary NID. It can also delete -peer's NIDs which Lustre told it about (not the Primary NID). - -If a peer has been manually discovered via - lnetctl discover -command, then make sure to delete the manually discovered -peer and recreate it with the Lustre NID information -provided for us. - -Lustre-change: https://review.whamcloud.com/50106 -Lustre-commit: aacb16191a72bc6db1155030849efb0d6971a572 - -Signed-off-by: Amir Shehata -Change-Id: I8fc8a69caccca047e3085bb33d026a3f09fb359b -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51130 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Frank Sehr -Reviewed-by: Cyril Bordage -Reviewed-by: Oleg Drokin ---- - lnet/lnet/peer.c | 113 +++++++++++++++++++++++++++++++++++++---------- - 1 file changed, 90 insertions(+), 23 deletions(-) - -diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c -index 7a438ea086..01eb9aa8aa 100644 ---- a/lnet/lnet/peer.c -+++ b/lnet/lnet/peer.c -@@ -546,6 +546,15 @@ lnet_peer_del_nid(struct lnet_peer *lp, lnet_nid_t nid4, unsigned int flags) - } - - lpni = lnet_peer_ni_find_locked(&nid); -+ /* If we're asked to lock down the primary NID we shouldn't be -+ * deleting it -+ */ -+ if (lp->lp_state & LNET_PEER_LOCK_PRIMARY && -+ nid_same(&primary_nid, &nid)) { -+ rc = -EPERM; -+ goto out; -+ } -+ - if (!lpni) { - rc = -ENOENT; - goto out; -@@ -1420,6 +1429,19 @@ LNetAddPeer(lnet_nid_t *nids, __u32 num_nids) - if (!pnid) { - pnid = nids[i]; - rc = lnet_add_peer_ni(pnid, LNET_NID_ANY, mr, true); -+ if (rc == -EALREADY) { -+ struct lnet_peer *lp; -+ -+ CDEBUG(D_NET, "A peer exists for NID %s\n", -+ libcfs_nid2str(pnid)); -+ rc = 0; -+ /* Adds a refcount */ -+ lp = lnet_find_peer4(pnid); -+ LASSERT(lp); -+ pnid = lnet_nid_to_nid4(&lp->lp_primary_nid); -+ /* Drop refcount from lookup */ -+ lnet_peer_decref_locked(lp); -+ } - } else if (lnet_peer_discovery_disabled) { - rc = lnet_add_peer_ni(nids[i], LNET_NID_ANY, mr, true); - } else { -@@ -1466,13 +1488,20 @@ LNetPrimaryNID(lnet_nid_t nid) - * down then this discovery can introduce long delays into the mount - * process, so skip it if it isn't necessary. - */ -- while (!lnet_peer_discovery_disabled && !lnet_peer_is_uptodate(lp)) { -- spin_lock(&lp->lp_lock); -+ spin_lock(&lp->lp_lock); -+ if (!lnet_peer_discovery_disabled && -+ (!(lp->lp_state & LNET_PEER_LOCK_PRIMARY) || -+ !lnet_peer_is_uptodate_locked(lp))) { - /* force a full discovery cycle */ -- lp->lp_state |= LNET_PEER_FORCE_PING | LNET_PEER_FORCE_PUSH; -+ lp->lp_state |= LNET_PEER_FORCE_PING | LNET_PEER_FORCE_PUSH | -+ LNET_PEER_LOCK_PRIMARY; - spin_unlock(&lp->lp_lock); - -- rc = lnet_discover_peer_locked(lpni, cpt, true); -+ /* start discovery in the background. Messages to that -+ * peer will not go through until the discovery is -+ * complete -+ */ -+ rc = lnet_discover_peer_locked(lpni, cpt, false); - if (rc) - goto out_decref; - /* The lpni (or lp) for this NID may have changed and our ref is -@@ -1486,14 +1515,8 @@ LNetPrimaryNID(lnet_nid_t nid) - goto out_unlock; - } - lp = lpni->lpni_peer_net->lpn_peer; -- -- /* If we find that the peer has discovery disabled then we will -- * not modify whatever primary NID is currently set for this -- * peer. Thus, we can break out of this loop even if the peer -- * is not fully up to date. -- */ -- if (lnet_is_discovery_disabled(lp)) -- break; -+ } else { -+ spin_unlock(&lp->lp_lock); - } - primary_nid = lnet_nid_to_nid4(&lp->lp_primary_nid); - out_decref: -@@ -1530,9 +1553,9 @@ lnet_peer_get_net_locked(struct lnet_peer *peer, __u32 net_id) - */ - static int - lnet_peer_attach_peer_ni(struct lnet_peer *lp, -- struct lnet_peer_net *lpn, -- struct lnet_peer_ni *lpni, -- unsigned flags) -+ struct lnet_peer_net *lpn, -+ struct lnet_peer_ni *lpni, -+ unsigned flags) - { - struct lnet_peer_table *ptable; - bool new_lpn = false; -@@ -1599,6 +1622,8 @@ lnet_peer_attach_peer_ni(struct lnet_peer *lp, - lnet_peer_clr_non_mr_pref_nids(lp); - } - } -+ if (flags & LNET_PEER_LOCK_PRIMARY) -+ lp->lp_state |= LNET_PEER_LOCK_PRIMARY; - spin_unlock(&lp->lp_lock); - - lp->lp_nnis++; -@@ -1661,13 +1686,28 @@ lnet_peer_add(lnet_nid_t nid4, unsigned int flags) - else if ((lp->lp_state ^ flags) & LNET_PEER_MULTI_RAIL) - rc = -EPERM; - goto out; -- } else if (!(flags & LNET_PEER_CONFIGURED)) { -- if (lnet_nid_to_nid4(&lp->lp_primary_nid) == nid4) { -+ } else if (lp->lp_state & LNET_PEER_LOCK_PRIMARY) { -+ if (nid_same(&lp->lp_primary_nid, &nid)) { - rc = -EEXIST; - goto out; - } -+ /* we're trying to recreate an existing peer which -+ * has already been created and its primary -+ * locked. This is likely due to two servers -+ * existing on the same node. So we'll just refer -+ * to that node with the primary NID which was -+ * first added by Lustre -+ */ -+ rc = -EALREADY; -+ goto out; - } -- /* Delete and recreate as a configured peer. */ -+ /* Delete and recreate the peer. -+ * We can get here: -+ * 1. If the peer is being recreated as a configured NID -+ * 2. if there already exists a peer which -+ * was discovered manually, but is recreated via Lustre -+ * with PRIMARY_lock -+ */ - rc = lnet_peer_del(lp); - if (rc) - goto out; -@@ -1760,9 +1800,27 @@ lnet_peer_add_nid(struct lnet_peer *lp, lnet_nid_t nid4, unsigned int flags) - } - /* If this is the primary NID, destroy the peer. */ - if (lnet_peer_ni_is_primary(lpni)) { -- struct lnet_peer *rtr_lp = -+ struct lnet_peer *lp2 = - lpni->lpni_peer_net->lpn_peer; -- int rtr_refcount = rtr_lp->lp_rtr_refcount; -+ int rtr_refcount = lp2->lp_rtr_refcount; -+ -+ /* If the new peer that this NID belongs to is -+ * a primary NID for another peer which we're -+ * suppose to preserve the Primary for then we -+ * don't want to mess with it. But the -+ * configuration is wrong at this point, so we -+ * should flag both of these peers as in a bad -+ * state -+ */ -+ if (lp2->lp_state & LNET_PEER_LOCK_PRIMARY) { -+ spin_lock(&lp->lp_lock); -+ lp->lp_state |= LNET_PEER_BAD_CONFIG; -+ spin_unlock(&lp->lp_lock); -+ spin_lock(&lp2->lp_lock); -+ lp2->lp_state |= LNET_PEER_BAD_CONFIG; -+ spin_unlock(&lp2->lp_lock); -+ goto out_free_lpni; -+ } - /* - * if we're trying to delete a router it means - * we're moving this peer NI to a new peer so must -@@ -1770,9 +1828,9 @@ lnet_peer_add_nid(struct lnet_peer *lp, lnet_nid_t nid4, unsigned int flags) - */ - if (rtr_refcount > 0) { - flags |= LNET_PEER_RTR_NI_FORCE_DEL; -- lnet_rtr_transfer_to_peer(rtr_lp, lp); -+ lnet_rtr_transfer_to_peer(lp2, lp); - } -- lnet_peer_del(lpni->lpni_peer_net->lpn_peer); -+ lnet_peer_del(lp2); - lnet_peer_ni_decref_locked(lpni); - lpni = lnet_peer_ni_alloc(&nid); - if (!lpni) { -@@ -1830,7 +1888,8 @@ lnet_peer_set_primary_nid(struct lnet_peer *lp, lnet_nid_t nid, - if (lnet_nid_to_nid4(&lp->lp_primary_nid) == nid) - goto out; - -- lnet_nid4_to_nid(nid, &lp->lp_primary_nid); -+ if (!(lp->lp_state & LNET_PEER_LOCK_PRIMARY)) -+ lnet_nid4_to_nid(nid, &lp->lp_primary_nid); - - rc = lnet_peer_add_nid(lp, nid, flags); - if (rc) { -@@ -1838,6 +1897,14 @@ lnet_peer_set_primary_nid(struct lnet_peer *lp, lnet_nid_t nid, - goto out; - } - out: -+ /* if this is a configured peer or the primary for that peer has -+ * been locked, then we don't want to flag this scenario as -+ * a failure -+ */ -+ if (lp->lp_state & LNET_PEER_CONFIGURED || -+ lp->lp_state & LNET_PEER_LOCK_PRIMARY) -+ return 0; -+ - CDEBUG(D_NET, "peer %s NID %s: %d\n", - libcfs_nidstr(&old), libcfs_nid2str(nid), rc); - --- -2.33.0 - diff --git a/0054-LU-14992-tests-sanity-replay-vbr-mkdir-on-MDT0.patch b/0015-LU-14992-tests-sanity-replay-vbr-mkdir-on-MDT0.patch similarity index 90% rename from 0054-LU-14992-tests-sanity-replay-vbr-mkdir-on-MDT0.patch rename to 0015-LU-14992-tests-sanity-replay-vbr-mkdir-on-MDT0.patch index 66ec0a25a3813268d32a6492c693ab171071b736..0c86b8fb76d2c004037f0596a96b7f78dc33545a 100644 --- a/0054-LU-14992-tests-sanity-replay-vbr-mkdir-on-MDT0.patch +++ b/0015-LU-14992-tests-sanity-replay-vbr-mkdir-on-MDT0.patch @@ -1,24 +1,27 @@ -From 24eaacb00262fe64001255f274f28e8a3837349a Mon Sep 17 00:00:00 2001 +From b71ad0c466c4e5a69e55af7f20a37bf4c2b883ec Mon Sep 17 00:00:00 2001 From: James Nunez Date: Mon, 13 Sep 2021 10:35:30 -0600 -Subject: [PATCH 54/61] LU-14992 tests: sanity/replay-vbr mkdir on MDT0 +Subject: [PATCH 15/49] LU-14992 tests: sanity/replay-vbr mkdir on MDT0 Replace mkdir with mkdir_on_mdt0() for sanity test 133a and relay-vbr test 7a. These tests expect the newly created directory is on MDT0. +Lustre-change: https://review.whamcloud.com/44902 +Lustre-commit: f0324c5c2f4390d6d7e93ed799e95d8eef4704f4 + Test-Parameters: trivial mdscount=2 mdtcount=4 testlist=sanity Test-Parameters: env=SLOW=yes mdscount=2 mdtcount=4 testlist=replay-vbr Signed-off-by: James Nunez Change-Id: Icea2923a8d8d3a3aa0ddf0401f0a025480b2f6f0 -Reviewed-on: https://review.whamcloud.com/44902 -Tested-by: jenkins -Tested-by: Maloo Reviewed-by: Kevin Zhao Reviewed-by: Andreas Dilger -Reviewed-by: Oleg Drokin Signed-off-by: Xinliang Liu -(cherry picked from commit f0324c5c2f4390d6d7e93ed799e95d8eef4704f4) +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49404 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: James Simmons +Reviewed-by: Oleg Drokin --- lustre/tests/replay-vbr.sh | 26 +++++++++++------------ lustre/tests/sanity.sh | 42 +++++++++++++++++++++++++------------- @@ -64,10 +67,10 @@ index 4c0f697494..2c785c459e 100755 run_test 7a "create, {lost}, create" diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh -index a004738552..3ebb944ab2 100755 +index a6f75b2328..41959033e8 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh -@@ -14092,21 +14092,31 @@ check_stats() { +@@ -14109,21 +14109,31 @@ check_stats() { local want=${3:-0} local res @@ -108,7 +111,7 @@ index a004738552..3ebb944ab2 100755 fi } -@@ -14126,8 +14136,11 @@ test_133a() { +@@ -14143,8 +14153,11 @@ test_133a() { do_facet ost1 $LCTL set_param obdfilter.*.stats=clear # verify mdt stats first. @@ -121,7 +124,7 @@ index a004738552..3ebb944ab2 100755 touch ${testdir}/${tfile} || error "touch failed" check_stats $SINGLEMDS "open" 1 check_stats $SINGLEMDS "close" 1 -@@ -14167,7 +14180,8 @@ test_133b() { +@@ -14184,7 +14197,8 @@ test_133b() { local testdir=$DIR/${tdir}/stats_testdir @@ -131,7 +134,7 @@ index a004738552..3ebb944ab2 100755 touch ${testdir}/${tfile} || error "touch failed" cancel_lru_locks mdc -@@ -14322,12 +14336,12 @@ test_133d() { +@@ -14339,12 +14353,12 @@ test_133d() { local testdir1=$DIR/${tdir}/stats_testdir1 local testdir2=$DIR/${tdir}/stats_testdir2 diff --git a/0016-LU-14668-lnet-Peers-added-via-kernel-API-should-be-p.patch b/0016-LU-14668-lnet-Peers-added-via-kernel-API-should-be-p.patch deleted file mode 100644 index 30b717fc43fd506df0f0bff79ec4178dbbdafb5a..0000000000000000000000000000000000000000 --- a/0016-LU-14668-lnet-Peers-added-via-kernel-API-should-be-p.patch +++ /dev/null @@ -1,167 +0,0 @@ -From f63e87f0a88a856d5cc38039afef704676ff5521 Mon Sep 17 00:00:00 2001 -From: Chris Horn -Date: Tue, 25 May 2021 11:17:49 -0500 -Subject: [PATCH 16/61] LU-14668 lnet: Peers added via kernel API should be - permanent - -The LNetAddPeer() API allows Lustre to predefine the Peer for LNet. -Originally these peers would be temporary and potentially re-created -via discovery. Instead, let's make these peers permanent. This allows -Lustre to dictate the primary NID of the peer. LNet makes sure this -primary NID is not changed afterwards. - -Lustre-change: https://review.whamcloud.com/43788 -Lustre-commit: 41733dadd8ad0e87e44dd19e25e576e90484cb9b - -Test-Parameters: trivial -Signed-off-by: Amir Shehata -Signed-off-by: Chris Horn -Change-Id: I3f54c04719c9e0374176682af08183f0c93ef737 -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51131 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Frank Sehr -Reviewed-by: Cyril Bordage -Reviewed-by: Oleg Drokin ---- - lnet/include/lnet/lib-lnet.h | 2 +- - lnet/lnet/api-ni.c | 7 +++--- - lnet/lnet/peer.c | 43 ++++++++++++++++++++++-------------- - 3 files changed, 32 insertions(+), 20 deletions(-) - -diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h -index 223c6d328b..a1261d137d 100644 ---- a/lnet/include/lnet/lib-lnet.h -+++ b/lnet/include/lnet/lib-lnet.h -@@ -1011,8 +1011,8 @@ void lnet_peer_clr_pref_rtrs(struct lnet_peer_ni *lpni); - int lnet_peer_add_pref_rtr(struct lnet_peer_ni *lpni, struct lnet_nid *nid); - int lnet_peer_ni_set_non_mr_pref_nid(struct lnet_peer_ni *lpni, - struct lnet_nid *nid); --int lnet_add_peer_ni(lnet_nid_t key_nid, lnet_nid_t nid, bool mr, bool temp); - int lnet_del_peer_ni(lnet_nid_t key_nid, lnet_nid_t nid); -+int lnet_user_add_peer_ni(lnet_nid_t prim_nid, struct lnet_nid *nid, bool mr); - int lnet_get_peer_info(struct lnet_ioctl_peer_cfg *cfg, void __user *bulk); - int lnet_get_peer_ni_info(__u32 peer_index, __u64 *nid, - char alivness[LNET_MAX_STR_LEN], -diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c -index b99c85b73d..08c28d5aac 100644 ---- a/lnet/lnet/api-ni.c -+++ b/lnet/lnet/api-ni.c -@@ -4204,9 +4204,10 @@ LNetCtl(unsigned int cmd, void *arg) - return -EINVAL; - - mutex_lock(&the_lnet.ln_api_mutex); -- rc = lnet_add_peer_ni(cfg->prcfg_prim_nid, -- cfg->prcfg_cfg_nid, -- cfg->prcfg_mr, false); -+ lnet_nid4_to_nid(cfg->prcfg_cfg_nid, &nid); -+ rc = lnet_user_add_peer_ni(cfg->prcfg_prim_nid, -+ &nid, -+ cfg->prcfg_mr); - mutex_unlock(&the_lnet.ln_api_mutex); - return rc; - } -diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c -index 01eb9aa8aa..040fa36d34 100644 ---- a/lnet/lnet/peer.c -+++ b/lnet/lnet/peer.c -@@ -47,6 +47,8 @@ - #define LNET_REDISCOVER_PEER (1) - - static int lnet_peer_queue_for_discovery(struct lnet_peer *lp); -+static int lnet_add_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, bool mr, -+ unsigned int flags); - - static void - lnet_peer_remove_from_remote_list(struct lnet_peer_ni *lpni) -@@ -1428,7 +1430,8 @@ LNetAddPeer(lnet_nid_t *nids, __u32 num_nids) - - if (!pnid) { - pnid = nids[i]; -- rc = lnet_add_peer_ni(pnid, LNET_NID_ANY, mr, true); -+ rc = lnet_add_peer_ni(pnid, LNET_NID_ANY, mr, -+ LNET_PEER_LOCK_PRIMARY); - if (rc == -EALREADY) { - struct lnet_peer *lp; - -@@ -1443,9 +1446,11 @@ LNetAddPeer(lnet_nid_t *nids, __u32 num_nids) - lnet_peer_decref_locked(lp); - } - } else if (lnet_peer_discovery_disabled) { -- rc = lnet_add_peer_ni(nids[i], LNET_NID_ANY, mr, true); -+ rc = lnet_add_peer_ni(nids[i], LNET_NID_ANY, mr, -+ LNET_PEER_LOCK_PRIMARY); - } else { -- rc = lnet_add_peer_ni(pnid, nids[i], mr, true); -+ rc = lnet_add_peer_ni(pnid, nids[i], mr, -+ LNET_PEER_LOCK_PRIMARY); - } - - if (rc && rc != -EEXIST) -@@ -1977,20 +1982,18 @@ out: - * The caller must hold ln_api_mutex. This prevents the peer from - * being created/modified/deleted by a different thread. - */ --int --lnet_add_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, bool mr, bool temp) -+static int -+lnet_add_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, bool mr, -+ unsigned int flags) -+__must_hold(&the_lnet.ln_api_mutex) - { - struct lnet_peer *lp = NULL; - struct lnet_peer_ni *lpni; -- unsigned int flags = 0; - - /* The prim_nid must always be specified */ - if (prim_nid == LNET_NID_ANY) - return -EINVAL; - -- if (!temp) -- flags = LNET_PEER_CONFIGURED; -- - if (mr) - flags |= LNET_PEER_MULTI_RAIL; - -@@ -2008,13 +2011,6 @@ lnet_add_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, bool mr, bool temp) - lnet_peer_ni_decref_locked(lpni); - lp = lpni->lpni_peer_net->lpn_peer; - -- /* Peer must have been configured. */ -- if (!temp && !(lp->lp_state & LNET_PEER_CONFIGURED)) { -- CDEBUG(D_NET, "peer %s was not configured\n", -- libcfs_nid2str(prim_nid)); -- return -ENOENT; -- } -- - /* Primary NID must match */ - if (lnet_nid_to_nid4(&lp->lp_primary_nid) != prim_nid) { - CDEBUG(D_NET, "prim_nid %s is not primary for peer %s\n", -@@ -2030,9 +2026,24 @@ lnet_add_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, bool mr, bool temp) - return -EPERM; - } - -+ if ((flags & LNET_PEER_LOCK_PRIMARY) && -+ (lnet_peer_is_uptodate(lp) && -+ (lp->lp_state & LNET_PEER_LOCK_PRIMARY))) { -+ CDEBUG(D_NET, -+ "Don't add temporary peer NI for uptodate peer %s\n", -+ libcfs_nidstr(&lp->lp_primary_nid)); -+ return -EINVAL; -+ } -+ - return lnet_peer_add_nid(lp, nid, flags); - } - -+int lnet_user_add_peer_ni(lnet_nid_t prim_nid, struct lnet_nid *nid, bool mr) -+{ -+ return lnet_add_peer_ni(prim_nid, lnet_nid_to_nid4(nid), mr, -+ LNET_PEER_CONFIGURED); -+} -+ - /* - * Implementation of IOC_LIBCFS_DEL_PEER_NI. - * --- -2.33.0 - diff --git a/0016-LU-17226-build-create-config-option-for-l_getsepol.patch b/0016-LU-17226-build-create-config-option-for-l_getsepol.patch new file mode 100644 index 0000000000000000000000000000000000000000..4bffa6ab1c0ea410708fe36db2f65757b1bad2f9 --- /dev/null +++ b/0016-LU-17226-build-create-config-option-for-l_getsepol.patch @@ -0,0 +1,108 @@ +From e12e831794cd172ba687567b7b50548740d83b49 Mon Sep 17 00:00:00 2001 +From: Gian-Carlo DeFazio +Date: Thu, 16 Nov 2023 15:05:45 -0800 +Subject: [PATCH 16/49] LU-17226 build: create config option for l_getsepol + +Add a configuration option for l_getsepol. +l_getsepol is build by default unless the --disable-l_getsepol +option is given to configure. +lustre.spec.in builds l_getsepol by default and has its +dependencies as build requirements. + +The implicit configuration check for the dependency +openssl-devel is removed and replaced by a BuildRequires. + +Lustre-change: https://review.whamcloud.com/52849 +Lustre-commit: 2777adcabd1032ddb886f913fa04d82a292ab379 + +Test-Parameters: trivial +Signed-off-by: Gian-Carlo DeFazio +Change-Id: If71a2a4a524047edbd2b31e6fac7a42f36a030bf +Reviewed-by: Sebastien Buisson +Reviewed-by: Andreas Dilger +Reviewed-by: Oleg Drokin +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54190 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Olaf Faaland +--- + config/lustre-build.m4 | 1 + + lustre.spec.in | 5 +++++ + lustre/autoconf/lustre-core.m4 | 12 +++++++++++- + 3 files changed, 17 insertions(+), 1 deletion(-) + +diff --git a/config/lustre-build.m4 b/config/lustre-build.m4 +index 28eb3ecfe8..a81ce2749b 100644 +--- a/config/lustre-build.m4 ++++ b/config/lustre-build.m4 +@@ -693,6 +693,7 @@ AS_IF([test "x$enable_utils" = xno], [enable_tests="no"]) + + AS_IF([test "x$enable_utils" = xyes], [ + LC_OPENSSL_GETSEPOL ++ LC_CONFIG_GETSEPOL + LC_FID2PATH_ANON_UNION + ]) + AS_IF([test "x$enable_tests" = xyes], [ +diff --git a/lustre.spec.in b/lustre.spec.in +index f4ac0c894d..2678d608db 100644 +--- a/lustre.spec.in ++++ b/lustre.spec.in +@@ -18,6 +18,7 @@ + %bcond_with mofed + %bcond_without mpi + %bcond_with kabi ++%bcond_without l_getsepol + + # LUTF Turn off brp-python-precompile script as we don't want the python files + # to be compiled on installation +@@ -224,6 +225,9 @@ Provides: lustre-client = %{version}-%{release} + #suse don't support selinux + BuildRequires: libselinux-devel + %endif ++%if %{with l_getsepol} ++BuildRequires: pkgconfig(libselinux) openssl-devel ++%endif + %if %{with lustre_modules} + %if %{with mofed} + BuildRequires: mlnx-ofa_kernel-devel +@@ -537,6 +541,7 @@ fi + %{!?with_systemd:--with-systemdsystemunitdir=no} \ + %{?with_systemd:--with-systemdsystemunitdir=%{_unitdir}} \ + %{?with_mofed:--with-o2ib=$o2ib_path} \ ++ %{!?with_l_getsepol:--disable-l_getsepol} \ + --with-linux=%{kdir} \ + --with-linux-obj=%{kobjdir} \ + --with-kmp-moddir=%{kmoddir}/%{name} +diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 +index 064347b8c0..8e32757774 100644 +--- a/lustre/autoconf/lustre-core.m4 ++++ b/lustre/autoconf/lustre-core.m4 +@@ -499,6 +499,15 @@ AC_MSG_RESULT([$enable_getsepol]) + CFLAGS="$saved_flags" + ]) # LC_OPENSSL_GETSEPOL + ++# LC_GCONFIG_GETSEPOL ++AC_DEFUN([LC_CONFIG_GETSEPOL], [ ++AC_ARG_ENABLE([l_getsepol], [AS_HELP_STRING([--disable-l_getsepol], ++ [build the l_getsepol utility])], [config_getsepol="no"], ++ [config_getsepol="yes"]) ++AC_MSG_CHECKING([whether to build l_getsepol]) ++AC_MSG_RESULT([$config_getsepol]) ++]) # LC_GETSEPOL ++ + # LC_HAVE_LIBAIO + AC_DEFUN([LC_HAVE_LIBAIO], [ + AC_CHECK_HEADER([libaio.h], +@@ -4312,7 +4321,8 @@ AM_CONDITIONAL(HAVE_SYSTEMD, test "x$with_systemdsystemunitdir" != "xno") + AM_CONDITIONAL(ENABLE_BASH_COMPLETION, test "x$with_bash_completion_dir" != "xno") + AM_CONDITIONAL(XATTR_HANDLER, test "x$lb_cv_compile_xattr_handler_flags" = xyes) + AM_CONDITIONAL(SELINUX, test "$SELINUX" = "-lselinux") +-AM_CONDITIONAL(GETSEPOL, test x$enable_getsepol = xyes) ++AM_CONDITIONAL(GETSEPOL, test x$enable_getsepol = xyes && ++ test x$config_getsepol = xyes) + AM_CONDITIONAL(LLCRYPT, test x$enable_llcrypt = xyes) + AM_CONDITIONAL(LIBAIO, test x$enable_libaio = xyes) + ]) # LC_CONDITIONALS +-- +2.33.0 + diff --git a/0017-LU-14668-lnet-don-t-delete-peer-created-by-Lustre.patch b/0017-LU-14668-lnet-don-t-delete-peer-created-by-Lustre.patch deleted file mode 100644 index 9e6098db4ef95024c4e2b76ed6acb71454c7247b..0000000000000000000000000000000000000000 --- a/0017-LU-14668-lnet-don-t-delete-peer-created-by-Lustre.patch +++ /dev/null @@ -1,98 +0,0 @@ -From 26d11f254795a2869ae30a7e5d6ebf2bee59f879 Mon Sep 17 00:00:00 2001 -From: Amir Shehata -Date: Wed, 5 May 2021 23:02:22 -0700 -Subject: [PATCH 17/61] LU-14668 lnet: don't delete peer created by Lustre - -Peers created by Lustre have their primary NIDs locked. -If that peer is deleted, it'll confuse lustre. So when manually -deleting a peer using: - lnetctl peer del --prim_nid ... -We must continue to preserve the primary NID. Therefore we delete -all the constituent NIDs, but keep the primary NID. We then -flag the peer for rediscovery. - -Lustre-change: https://review.whamcloud.com/43565 -Lustre-commit: 7cc5b4329fc2eecbf09dbda85efe58f4ad5a32b9 - -Signed-off-by: Amir Shehata -Change-Id: I34eef9b0049435a01fde87dc8263dd50f631c551 -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51132 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Frank Sehr -Reviewed-by: Cyril Bordage -Reviewed-by: Oleg Drokin ---- - lnet/lnet/peer.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++-- - 1 file changed, 47 insertions(+), 2 deletions(-) - -diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c -index 040fa36d34..3b45639b19 100644 ---- a/lnet/lnet/peer.c -+++ b/lnet/lnet/peer.c -@@ -2044,6 +2044,43 @@ int lnet_user_add_peer_ni(lnet_nid_t prim_nid, struct lnet_nid *nid, bool mr) - LNET_PEER_CONFIGURED); - } - -+static int -+lnet_reset_peer(struct lnet_peer *lp) -+{ -+ struct lnet_peer_net *lpn, *lpntmp; -+ struct lnet_peer_ni *lpni, *lpnitmp; -+ unsigned int flags; -+ int rc; -+ -+ lnet_peer_cancel_discovery(lp); -+ -+ flags = LNET_PEER_CONFIGURED; -+ if (lp->lp_state & LNET_PEER_MULTI_RAIL) -+ flags |= LNET_PEER_MULTI_RAIL; -+ -+ list_for_each_entry_safe(lpn, lpntmp, &lp->lp_peer_nets, -+ lpn_peer_nets) { -+ list_for_each_entry_safe(lpni, lpnitmp, &lpn->lpn_peer_nis, -+ lpni_peer_nis) { -+ if (nid_same(&lpni->lpni_nid, &lp->lp_primary_nid)) -+ continue; -+ -+ rc = lnet_peer_del_nid(lp, -+ lnet_nid_to_nid4(&lpni->lpni_nid), -+ flags); -+ if (rc) { -+ CERROR("Failed to delete %s from peer %s\n", -+ libcfs_nidstr(&lpni->lpni_nid), -+ libcfs_nidstr(&lp->lp_primary_nid)); -+ } -+ } -+ } -+ -+ /* mark it for discovery the next time we use it */ -+ lp->lp_state &= ~LNET_PEER_NIDS_UPTODATE; -+ return 0; -+} -+ - /* - * Implementation of IOC_LIBCFS_DEL_PEER_NI. - * -@@ -2087,8 +2124,16 @@ lnet_del_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid) - } - lnet_net_unlock(LNET_LOCK_EX); - -- if (nid == LNET_NID_ANY || nid == lnet_nid_to_nid4(&lp->lp_primary_nid)) -- return lnet_peer_del(lp); -+ if (nid == LNET_NID_ANY || -+ nid == lnet_nid_to_nid4(&lp->lp_primary_nid)) { -+ if (lp->lp_state & LNET_PEER_LOCK_PRIMARY) { -+ CERROR("peer %s created by Lustre. Must preserve primary NID, but will remove other NIDs\n", -+ libcfs_nidstr(&lp->lp_primary_nid)); -+ return lnet_reset_peer(lp); -+ } else { -+ return lnet_peer_del(lp); -+ } -+ } - - flags = LNET_PEER_CONFIGURED; - if (lp->lp_state & LNET_PEER_MULTI_RAIL) --- -2.33.0 - diff --git a/0017-LU-17664-lnet-LNetPrimaryNID-returns-wrong-NID.patch b/0017-LU-17664-lnet-LNetPrimaryNID-returns-wrong-NID.patch new file mode 100644 index 0000000000000000000000000000000000000000..769fbfeadccad2c8220cea0cdd6f2e8c85fdbf50 --- /dev/null +++ b/0017-LU-17664-lnet-LNetPrimaryNID-returns-wrong-NID.patch @@ -0,0 +1,64 @@ +From 4937c3ccca8a3a3f8d5fdd2e5007d65773fcadc5 Mon Sep 17 00:00:00 2001 +From: Chris Horn +Date: Fri, 22 Mar 2024 15:02:55 -0600 +Subject: [PATCH 17/49] LU-17664 lnet: LNetPrimaryNID returns wrong NID + +This is a regression introduced by the b2_15 port of + +Lustre-change: https://review.whamcloud.com/50159 +Lustre-commit: fc7a0d6013b46ebc17cdfdccc04a5d1d92c6af24 + +This port assigned the return value to the wrong variable. As a +result, LNetPrimaryNID() will always return the same value that +was passed in as an argument. Fix the issue by assigning the +return value to the correct variable. + +Test-Parameters: trivial +Fixes: 6cfc8e55a2 ("LU-14668 lnet: add 'lock_prim_nid" lnet module parameter") +Signed-off-by: Chris Horn +Change-Id: I10e334d38e8c257239ae3fa82d0aea4f87d8ba69 +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54537 +Reviewed-by: Serguei Smirnov +Reviewed-by: Andreas Dilger +Reviewed-by: Cyril Bordage +Reviewed-by: Frank Sehr +Reviewed-by: Oleg Drokin +Tested-by: jenkins +Tested-by: Maloo +--- + lnet/lnet/peer.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c +index 4eb7a44cf4..4ba681abbc 100644 +--- a/lnet/lnet/peer.c ++++ b/lnet/lnet/peer.c +@@ -1501,14 +1501,14 @@ again: + + /* DD disabled, nothing to do */ + if (lnet_peer_discovery_disabled) { +- nid = lnet_nid_to_nid4(&lp->lp_primary_nid); ++ primary_nid = lnet_nid_to_nid4(&lp->lp_primary_nid); + spin_unlock(&lp->lp_lock); + goto out_decref; + } + + /* Peer already up to date, nothing to do */ + if (lnet_peer_is_uptodate_locked(lp)) { +- nid = lnet_nid_to_nid4(&lp->lp_primary_nid); ++ primary_nid = lnet_nid_to_nid4(&lp->lp_primary_nid); + spin_unlock(&lp->lp_lock); + goto out_decref; + } +@@ -1541,7 +1541,7 @@ again: + + if (!lock_prim_nid && !lnet_is_discovery_disabled(lp)) + goto again; +- nid = lnet_nid_to_nid4(&lp->lp_primary_nid); ++ primary_nid = lnet_nid_to_nid4(&lp->lp_primary_nid); + out_decref: + lnet_peer_ni_decref_locked(lpni); + out_unlock: +-- +2.33.0 + diff --git a/0018-LU-14668-lnet-add-force-option-to-lnetctl-peer-del.patch b/0018-LU-14668-lnet-add-force-option-to-lnetctl-peer-del.patch deleted file mode 100644 index e007e314e07faf4f96a18a8c0aee7809d48cf57f..0000000000000000000000000000000000000000 --- a/0018-LU-14668-lnet-add-force-option-to-lnetctl-peer-del.patch +++ /dev/null @@ -1,375 +0,0 @@ -From 8c4df87ec21bf5d61dab4b6580fc7f7ecfa91e37 Mon Sep 17 00:00:00 2001 -From: Serguei Smirnov -Date: Mon, 27 Feb 2023 15:41:19 -0800 -Subject: [PATCH 18/61] LU-14668 lnet: add 'force' option to lnetctl peer del - -Add --force option to 'lnetctl peer del' command. -If the peer has primary NID locked, this option allows -for the peer to be deleted manually: - lnetctl peer del --prim_nid --force - -Add --prim_lock option to 'lnetctl peer add' command. -If specified, the primary NID of the peer is locked -such that it is going to be the NID used to identify -the peer in communications with Lustre layer. - -Lustre-change: https://review.whamcloud.com/50149 -Lustre-commit: f1b2d8d60c593a670b36006bcf9b040549d8c13a - -Test-Parameters: trivial -Signed-off-by: Serguei Smirnov -Change-Id: Ia6001856cfbce7b0c3288cff9b244b569d259647 -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51133 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Frank Sehr -Reviewed-by: Cyril Bordage -Reviewed-by: Oleg Drokin ---- - lnet/include/lnet/lib-lnet.h | 5 +++-- - lnet/include/uapi/linux/lnet/lnet-dlc.h | 4 +++- - lnet/lnet/api-ni.c | 6 +++-- - lnet/lnet/peer.c | 12 +++++----- - lnet/utils/lnetconfig/liblnetconfig.c | 24 +++++++++++--------- - lnet/utils/lnetconfig/liblnetconfig.h | 5 +++-- - lnet/utils/lnetctl.c | 29 +++++++++++++++++++++---- - lustre/doc/lnetctl.8 | 6 +++++ - lustre/tests/sanity-lnet.sh | 18 +++++++++++++++ - 9 files changed, 83 insertions(+), 26 deletions(-) - -diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h -index a1261d137d..5805586fb2 100644 ---- a/lnet/include/lnet/lib-lnet.h -+++ b/lnet/include/lnet/lib-lnet.h -@@ -1011,8 +1011,9 @@ void lnet_peer_clr_pref_rtrs(struct lnet_peer_ni *lpni); - int lnet_peer_add_pref_rtr(struct lnet_peer_ni *lpni, struct lnet_nid *nid); - int lnet_peer_ni_set_non_mr_pref_nid(struct lnet_peer_ni *lpni, - struct lnet_nid *nid); --int lnet_del_peer_ni(lnet_nid_t key_nid, lnet_nid_t nid); --int lnet_user_add_peer_ni(lnet_nid_t prim_nid, struct lnet_nid *nid, bool mr); -+int lnet_del_peer_ni(lnet_nid_t key_nid, lnet_nid_t nid, int force); -+int lnet_user_add_peer_ni(lnet_nid_t prim_nid, struct lnet_nid *nid, bool mr, -+ bool lock_prim); - int lnet_get_peer_info(struct lnet_ioctl_peer_cfg *cfg, void __user *bulk); - int lnet_get_peer_ni_info(__u32 peer_index, __u64 *nid, - char alivness[LNET_MAX_STR_LEN], -diff --git a/lnet/include/uapi/linux/lnet/lnet-dlc.h b/lnet/include/uapi/linux/lnet/lnet-dlc.h -index 2b2c05fa3b..6718f34d93 100644 ---- a/lnet/include/uapi/linux/lnet/lnet-dlc.h -+++ b/lnet/include/uapi/linux/lnet/lnet-dlc.h -@@ -273,7 +273,9 @@ struct lnet_ioctl_peer_cfg { - struct libcfs_ioctl_hdr prcfg_hdr; - lnet_nid_t prcfg_prim_nid; - lnet_nid_t prcfg_cfg_nid; -- __u32 prcfg_count; -+ __u32 prcfg_count; /* ADD_PEER_NI: used for 'lock_prim' option -+ * DEL_PEER_NI: used for 'force' option -+ */ - __u32 prcfg_mr; - __u32 prcfg_state; - __u32 prcfg_size; -diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c -index 08c28d5aac..3a56ea83a1 100644 ---- a/lnet/lnet/api-ni.c -+++ b/lnet/lnet/api-ni.c -@@ -4207,7 +4207,8 @@ LNetCtl(unsigned int cmd, void *arg) - lnet_nid4_to_nid(cfg->prcfg_cfg_nid, &nid); - rc = lnet_user_add_peer_ni(cfg->prcfg_prim_nid, - &nid, -- cfg->prcfg_mr); -+ cfg->prcfg_mr, -+ cfg->prcfg_count == 1); - mutex_unlock(&the_lnet.ln_api_mutex); - return rc; - } -@@ -4220,7 +4221,8 @@ LNetCtl(unsigned int cmd, void *arg) - - mutex_lock(&the_lnet.ln_api_mutex); - rc = lnet_del_peer_ni(cfg->prcfg_prim_nid, -- cfg->prcfg_cfg_nid); -+ cfg->prcfg_cfg_nid, -+ cfg->prcfg_count); - mutex_unlock(&the_lnet.ln_api_mutex); - return rc; - } -diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c -index 3b45639b19..33a16f8470 100644 ---- a/lnet/lnet/peer.c -+++ b/lnet/lnet/peer.c -@@ -2038,10 +2038,12 @@ __must_hold(&the_lnet.ln_api_mutex) - return lnet_peer_add_nid(lp, nid, flags); - } - --int lnet_user_add_peer_ni(lnet_nid_t prim_nid, struct lnet_nid *nid, bool mr) -+int lnet_user_add_peer_ni(lnet_nid_t prim_nid, struct lnet_nid *nid, bool mr, -+ bool lock_prim) - { -- return lnet_add_peer_ni(prim_nid, lnet_nid_to_nid4(nid), mr, -- LNET_PEER_CONFIGURED); -+ int fl = LNET_PEER_CONFIGURED | (LNET_PEER_LOCK_PRIMARY * lock_prim); -+ -+ return lnet_add_peer_ni(prim_nid, lnet_nid_to_nid4(nid), mr, fl); - } - - static int -@@ -2093,7 +2095,7 @@ lnet_reset_peer(struct lnet_peer *lp) - * being modified/deleted by a different thread. - */ - int --lnet_del_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid) -+lnet_del_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid, int force) - { - struct lnet_peer *lp; - struct lnet_peer_ni *lpni; -@@ -2126,7 +2128,7 @@ lnet_del_peer_ni(lnet_nid_t prim_nid, lnet_nid_t nid) - - if (nid == LNET_NID_ANY || - nid == lnet_nid_to_nid4(&lp->lp_primary_nid)) { -- if (lp->lp_state & LNET_PEER_LOCK_PRIMARY) { -+ if (!force && lp->lp_state & LNET_PEER_LOCK_PRIMARY) { - CERROR("peer %s created by Lustre. Must preserve primary NID, but will remove other NIDs\n", - libcfs_nidstr(&lp->lp_primary_nid)); - return lnet_reset_peer(lp); -diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c -index 9f0dabd903..bbf5d3824a 100644 ---- a/lnet/utils/lnetconfig/liblnetconfig.c -+++ b/lnet/utils/lnetconfig/liblnetconfig.c -@@ -630,7 +630,7 @@ int lustre_lnet_discover_nid(char *ping_nids, int force, int seq_no, - } - - static int lustre_lnet_handle_peer_nidlist(lnet_nid_t *nidlist, int num_nids, -- bool is_mr, __u32 cmd, -+ bool is_mr, int option, __u32 cmd, - char *cmd_type, char *err_str) - { - struct lnet_ioctl_peer_cfg data; -@@ -645,6 +645,7 @@ static int lustre_lnet_handle_peer_nidlist(lnet_nid_t *nidlist, int num_nids, - data.prcfg_mr = is_mr; - data.prcfg_prim_nid = nidlist[0]; - data.prcfg_cfg_nid = LNET_NID_ANY; -+ data.prcfg_count = option; - - rc = dispatch_peer_ni_cmd(cmd, &data, err_str, cmd_type); - -@@ -660,6 +661,7 @@ static int lustre_lnet_handle_peer_nidlist(lnet_nid_t *nidlist, int num_nids, - data.prcfg_mr = is_mr; - data.prcfg_prim_nid = nidlist[0]; - data.prcfg_cfg_nid = nidlist[nid_idx]; -+ data.prcfg_count = option; - - rc = dispatch_peer_ni_cmd(cmd, &data, err_str, cmd_type); - -@@ -675,6 +677,7 @@ static int lustre_lnet_handle_peer_nidlist(lnet_nid_t *nidlist, int num_nids, - LIBCFS_IOC_INIT_V2(data, prcfg_hdr); - data.prcfg_prim_nid = nidlist[0]; - data.prcfg_cfg_nid = LNET_NID_ANY; -+ data.prcfg_count = option; - - rc = dispatch_peer_ni_cmd(cmd, &data, err_str, cmd_type); - } -@@ -684,8 +687,8 @@ static int lustre_lnet_handle_peer_nidlist(lnet_nid_t *nidlist, int num_nids, - - static int - lustre_lnet_mod_peer_nidlist(lnet_nid_t pnid, lnet_nid_t *lnet_nidlist, -- int cmd, int num_nids, bool is_mr, int seq_no, -- struct cYAML **err_rc) -+ int cmd, int num_nids, bool is_mr, int option, -+ int seq_no, struct cYAML **err_rc) - { - int rc = LUSTRE_CFG_RC_NO_ERR; - char err_str[LNET_MAX_STR_LEN]; -@@ -706,8 +709,8 @@ lustre_lnet_mod_peer_nidlist(lnet_nid_t pnid, lnet_nid_t *lnet_nidlist, - (num_nids - 1)); - - rc = lustre_lnet_handle_peer_nidlist(lnet_nidlist2, -- num_nids, is_mr, ioc_cmd, -- cmd_str, err_str); -+ num_nids, is_mr, option, -+ ioc_cmd, cmd_str, err_str); - out: - if (lnet_nidlist2) - free(lnet_nidlist2); -@@ -734,8 +737,8 @@ replace_sep(char *str, char sep, char newsep) - } - } - --int lustre_lnet_modify_peer(char *prim_nid, char *nids, bool is_mr, -- int cmd, int seq_no, struct cYAML **err_rc) -+int lustre_lnet_modify_peer(char *prim_nid, char *nids, bool is_mr, int cmd, -+ int option, int seq_no, struct cYAML **err_rc) - { - int num_nids, rc; - char err_str[LNET_MAX_STR_LEN] = "Error"; -@@ -774,7 +777,7 @@ int lustre_lnet_modify_peer(char *prim_nid, char *nids, bool is_mr, - - rc = lustre_lnet_mod_peer_nidlist(pnid, lnet_nidlist, - cmd, num_nids, is_mr, -- -1, err_rc); -+ option, -1, err_rc); - - out: - if (rc != LUSTRE_CFG_RC_NO_ERR) -@@ -4696,6 +4699,7 @@ static int handle_yaml_peer_common(struct cYAML *tree, struct cYAML **show_rc, - struct cYAML *seq_no, *prim_nid, *mr, *peer_nis; - lnet_nid_t lnet_nidlist[LNET_MAX_NIDS_PER_PEER]; - lnet_nid_t pnid = LNET_NID_ANY; -+ int force = 0; - - seq_no = cYAML_get_object_item(tree, "seq_no"); - seqn = seq_no ? seq_no->cy_valueint : -1; -@@ -4762,8 +4766,8 @@ static int handle_yaml_peer_common(struct cYAML *tree, struct cYAML **show_rc, - } - - rc = lustre_lnet_mod_peer_nidlist(pnid, lnet_nidlist, cmd, -- num_nids, mr_value, seqn, -- err_rc); -+ num_nids, mr_value, force, -+ seqn, err_rc); - - failed: - if (nidstr) -diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h -index 5096ac0c60..2c5397dce3 100644 ---- a/lnet/utils/lnetconfig/liblnetconfig.h -+++ b/lnet/utils/lnetconfig/liblnetconfig.h -@@ -664,11 +664,12 @@ int lustre_lnet_reset_stats(int seq_no, struct cYAML **err_rc); - * nids - a comma separated string of nids - * is_mr - Specifies whether this peer is MR capable. - * cmd - CONFIG or DELETE -+ * force - whether force-deleting a peer with locked primary nid - * seq_no - sequence number of the command - * err_rc - YAML structure of the resultant return code - */ --int lustre_lnet_modify_peer(char *prim_nid, char *nids, bool is_mr, -- int cmd, int seq_no, struct cYAML **err_rc); -+int lustre_lnet_modify_peer(char *prim_nid, char *nids, bool is_mr, int cmd, -+ int force, int seq_no, struct cYAML **err_rc); - - /* - * lustre_lnet_show_peer -diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c -index 51dfab605e..0f89e206e8 100644 ---- a/lnet/utils/lnetctl.c -+++ b/lnet/utils/lnetctl.c -@@ -258,12 +258,14 @@ command_t peer_cmds[] = { - "\t--prim_nid: Primary NID of the peer.\n" - "\t--nid: one or more peer NIDs\n" - "\t--non_mr: create this peer as not Multi-Rail capable\n" -- "\t--ip2nets: specify a range of nids per peer"}, -+ "\t--ip2nets: specify a range of nids per peer\n" -+ "\t--lock_prim: lock primary nid\n"}, - {"del", jt_del_peer_nid, 0, "delete a peer NID\n" - "\t--prim_nid: Primary NID of the peer.\n" - "\t--nid: list of NIDs to remove. If none provided,\n" - "\t peer is deleted\n" -- "\t--ip2nets: specify a range of nids per peer"}, -+ "\t--ip2nets: specify a range of nids per peer\n" -+ "\t--force: force-delete locked primary NID\n"}, - {"show", jt_show_peer, 0, "show peer information\n" - "\t--nid: NID of peer to filter on.\n" - "\t--verbose: display detailed output per peer." -@@ -2082,12 +2084,15 @@ static int jt_peer_nid_common(int argc, char **argv, int cmd) - char *prim_nid = NULL, *nidstr = NULL; - char err_str[LNET_MAX_STR_LEN] = "Error"; - struct cYAML *err_rc = NULL; -+ int force_lock = 0; - -- const char *const short_opts = "k:mn:"; -+ const char *const short_opts = "k:m:n:f:l"; - const struct option long_opts[] = { - { .name = "prim_nid", .has_arg = required_argument, .val = 'k' }, - { .name = "non_mr", .has_arg = no_argument, .val = 'm' }, - { .name = "nid", .has_arg = required_argument, .val = 'n' }, -+ { .name = "force", .has_arg = no_argument, .val = 'f' }, -+ { .name = "lock_prim", .has_arg = no_argument, .val = 'l' }, - { .name = NULL } }; - - rc = check_cmd(peer_cmds, "peer", "add", 2, argc, argv); -@@ -2112,6 +2117,22 @@ static int jt_peer_nid_common(int argc, char **argv, int cmd) - } - is_mr = false; - break; -+ case 'f': -+ if (cmd == LNETCTL_ADD_CMD) { -+ rc = LUSTRE_CFG_RC_BAD_PARAM; -+ snprintf(err_str, LNET_MAX_STR_LEN, -+ "Unrecognized option '-%c'", opt); -+ } -+ force_lock = 1; -+ break; -+ case 'l': -+ if (cmd == LNETCTL_DEL_CMD) { -+ rc = LUSTRE_CFG_RC_BAD_PARAM; -+ snprintf(err_str, LNET_MAX_STR_LEN, -+ "Unrecognized option '-%c'", opt); -+ } -+ force_lock = 1; -+ break; - case '?': - print_help(peer_cmds, "peer", - cmd == LNETCTL_ADD_CMD ? "add" : "del"); -@@ -2121,7 +2142,7 @@ static int jt_peer_nid_common(int argc, char **argv, int cmd) - } - - rc = lustre_lnet_modify_peer(prim_nid, nidstr, is_mr, cmd, -- -1, &err_rc); -+ force_lock, -1, &err_rc); - if (rc != LUSTRE_CFG_RC_NO_ERR) - goto out; - -diff --git a/lustre/doc/lnetctl.8 b/lustre/doc/lnetctl.8 -index f2a2793825..0df13e2493 100644 ---- a/lustre/doc/lnetctl.8 -+++ b/lustre/doc/lnetctl.8 -@@ -125,6 +125,9 @@ Configure an LNET peer with at least one supplied NID\. The primary NID must be - \-\-non_mr: create this peer as not Multi-Rail capable\. - . - .br -+\-\-lock_prim: lock primary NID of the peer for the purpose of identification with Lustre\. -+. -+.br - - .TP - \fBlnetctl peer\fR del -@@ -139,6 +142,9 @@ Delete a peer NID. The primary NID must be specified. If the removed NID is th - \-\-prim_nid: Primary NID of the peer\. - . - .br -+\-\-force: optional, use to delete a peer with primary NID locked\. -+. -+.br - - .TP - \fBlnetctl peer\fR show -diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh -index 367cb19d15..4af5e96ef8 100755 ---- a/lustre/tests/sanity-lnet.sh -+++ b/lustre/tests/sanity-lnet.sh -@@ -932,6 +932,24 @@ EOF - } - run_test 25 "Delete all secondary nids from peer (tcp, gni and o2ib)" - -+test_26() { -+ reinit_dlc || return $? -+ -+ do_lnetctl peer add --prim_nid 1.1.1.1@tcp --lock_prim || -+ error "Peer add with --lock_prim option failed $?" -+ do_lnetctl peer del --prim_nid 1.1.1.1@tcp || -+ error "Peer del failed $?" -+ $LNETCTL peer show --nid 1.1.1.1@tcp | grep -q 1.1.1.1@tcp || -+ error "1.1.1.1@tcp is not listed" -+ do_lnetctl peer del --prim_nid 1.1.1.1@tcp --force || -+ error "Peer del --force failed $?" -+ do_lnetctl peer show --nid 1.1.1.1@tcp && -+ error "failed to delete 1.1.1.1@tcp" -+ -+ return 0 -+} -+run_test 26 "Delete peer with primary nid locked" -+ - test_99a() { - reinit_dlc || return $? - --- -2.33.0 - diff --git a/0018-LU-16351-llite-Linux-6.1-prandom-folios_contig-vma_i.patch b/0018-LU-16351-llite-Linux-6.1-prandom-folios_contig-vma_i.patch new file mode 100644 index 0000000000000000000000000000000000000000..76166f4fbb3e513ef65e030004d3ba23e2112c88 --- /dev/null +++ b/0018-LU-16351-llite-Linux-6.1-prandom-folios_contig-vma_i.patch @@ -0,0 +1,631 @@ +From be2c7148733d7d772d38462a8da0644aff92b0cf Mon Sep 17 00:00:00 2001 +From: Shaun Tancheff +Date: Wed, 10 Apr 2024 00:45:22 -0700 +Subject: [PATCH 18/49] LU-16351 llite: Linux 6.1 prandom, folios_contig, + vma_iterator + +Linux commit v4.10-rc3-6-gc440408cf690 + random: convert get_random_int/long into get_random_u32/u64 +Linux commit v6.0-11338-gde492c83cae0 + prandom: remove unused functions + +prandom_u32 is a wrapper around get_random_u32, change users +of prandom_u32 to get_random_u32 and provide a fallback +to prandom_u32 when get_random_u32 is not available. + +Linux commit v6.0-rc1-2-g25885a35a720 + Change calling conventions for filldir_t +Add a test for the new filldir_t signature +Provide wrappers for transition from int (error code) to bool + +Linux commit v6.0-rc3-94-g35b471467f88 + filemap: add filemap_get_folios_contig() +Provide a wrapper and fallback to find_get_pages_contig + +Linux commit v6.0-rc3-225-gf39af05949a4 + mm: add VMA iterator +Use vma_iterator and for_each_vma when available. + +Lustre-change: https://review.whamcloud.com/49232 +Lustre-commit: ca992899d55fd13e65b75ace02931daaa29c18bd + +Test-Parameters: trivial +HPE-bug-id: LUS-11377 +Signed-off-by: Shaun Tancheff +Change-Id: I23dc23d0252e1995555b6685f5cf7c207edf642b +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54700 +Reviewed-by: Yang Sheng +Reviewed-by: Oleg Drokin +Tested-by: jenkins +Tested-by: Maloo +--- + contrib/scripts/spelling.txt | 3 +- + lustre/autoconf/lustre-core.m4 | 101 ++++++++++++++++++++++++++++++ + lustre/llite/llite_mmap.c | 22 +++++-- + lustre/llite/llite_nfs.c | 51 ++++++++++----- + lustre/llite/vvp_dev.c | 27 +++++++- + lustre/obdclass/llog_test.c | 2 +- + lustre/obdclass/lu_object.c | 2 +- + lustre/obdclass/lu_tgt_descs.c | 8 ++- + lustre/osd-ldiskfs/osd_handler.c | 8 ++- + lustre/osd-ldiskfs/osd_internal.h | 18 ++++++ + lustre/osd-ldiskfs/osd_scrub.c | 61 ++++++++++-------- + 11 files changed, 248 insertions(+), 55 deletions(-) + +diff --git a/contrib/scripts/spelling.txt b/contrib/scripts/spelling.txt +index 5e34d04572..c011c83e75 100644 +--- a/contrib/scripts/spelling.txt ++++ b/contrib/scripts/spelling.txt +@@ -84,7 +84,7 @@ CFS_PAGE_MASK||PAGE_MASK + CFS_PAGE_SIZE||PAGE_SIZE + cfs_proc_dir_entry_t||struct proc_dir_entry + cfs_rcu_head_t||struct rcu_head +-cfs_rand||prandom_u32 ++cfs_rand||get_random_u32 + cfs_srand||add_device_randomness + cfs_trimwhite||strim + cfs_time_add_64||ktime_add +@@ -197,6 +197,7 @@ PAGE_CACHE_MASK||PAGE_MASK + page_cache_release||put_page + PAGE_CACHE_SHIFT||PAGE_SHIFT + PAGE_CACHE_SIZE||PAGE_SIZE ++prandom_u32||get_random_u32 + return seq_printf||seq_printf + setup_timer||cfs_timer_setup + = seq_printf||seq_printf +diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 +index 8e32757774..dd1df4950e 100644 +--- a/lustre/autoconf/lustre-core.m4 ++++ b/lustre/autoconf/lustre-core.m4 +@@ -3454,6 +3454,97 @@ AC_DEFUN([LC_HAVE_GET_EXPIRY_TIME64_T], [ + ]) + ]) # LC_HAVE_GET_EXPIRY_TIME64_T + ++# ++# LC_HAVE_GET_RANDOM_U32_AND_U64 ++# ++# Linux commit v4.10-rc3-6-gc440408cf690 ++# random: convert get_random_int/long into get_random_u32/u64 ++# Linux commit v6.0-11338-gde492c83cae0 ++# prandom: remove unused functions ++# ++AC_DEFUN([LC_SRC_HAVE_GET_RANDOM_U32_AND_U64], [ ++ LB2_LINUX_TEST_SRC([get_random_u32_and_u64], [ ++ #include ++ ],[ ++ u32 rand32 = get_random_u32(); ++ u64 rand64 = get_random_u64(); ++ (void)rand32; ++ (void)rand64; ++ ],[-Werror]) ++]) ++AC_DEFUN([LC_HAVE_GET_RANDOM_U32_AND_U64], [ ++ AC_MSG_CHECKING([if get_random_u32() and get_random_u64() are available]) ++ LB2_LINUX_TEST_RESULT([get_random_u32_and_u64], [ ++ AC_DEFINE(HAVE_GET_RANDOM_U32_AND_U64, 1, ++ [get_random_[u32|u64] are available]) ++ ],[ ++ AC_DEFINE([get_random_u32()], [prandom_u32()], ++ [get_random_u32() is not available, use prandom_u32]) ++ ]) ++]) # LC_HAVE_GET_RANDOM_U32_AND_U64 ++ ++# ++# LC_NFS_FILLDIR_USE_CTX_RETURN_BOOL ++# ++# Linux commit v6.0-rc1-2-g25885a35a720 ++# Change calling conventions for filldir_t ++# ++AC_DEFUN([LC_SRC_NFS_FILLDIR_USE_CTX_RETURN_BOOL], [ ++ LB2_LINUX_TEST_SRC([filldir_ctx_return_bool], [ ++ #include ++ ],[ ++ bool filldir(struct dir_context *ctx, const char* name, ++ int i, loff_t off, u64 tmp, unsigned temp) ++ { ++ return 0; ++ } ++ ++ struct dir_context ctx = { ++ .actor = filldir, ++ }; ++ ++ ctx.actor(NULL, "test", 0, (loff_t) 0, 0, 0); ++ ],[-Werror]) ++]) ++AC_DEFUN([LC_NFS_FILLDIR_USE_CTX_RETURN_BOOL], [ ++ AC_MSG_CHECKING([if filldir_t uses struct dir_context and returns bool]) ++ LB2_LINUX_TEST_RESULT([filldir_ctx_return_bool], [ ++ AC_DEFINE(HAVE_FILLDIR_USE_CTX_RETURN_BOOL, 1, ++ [filldir_t needs struct dir_context and returns bool]) ++ AC_DEFINE(HAVE_FILLDIR_USE_CTX, 1, ++ [filldir_t needs struct dir_context as argument]) ++ AC_DEFINE(FILLDIR_TYPE, bool, ++ [filldir_t return type is bool or int]) ++ ],[ ++ AC_DEFINE(FILLDIR_TYPE, int, ++ [filldir_t return type is bool or int]) ++ ]) ++]) # LC_NFS_FILLDIR_USE_CTX_RETURN_BOOL ++ ++# ++# LC_HAVE_FILEMAP_GET_FOLIOS_CONTIG ++# ++# Linux commit v6.0-rc3-94-g35b471467f88 ++# filemap: add filemap_get_folios_contig() ++# ++AC_DEFUN([LC_SRC_HAVE_FILEMAP_GET_FOLIOS_CONTIG], [ ++ LB2_LINUX_TEST_SRC([filemap_get_folios_contig], [ ++ #include ++ ],[ ++ struct address_space *m = NULL; ++ pgoff_t start = 0; ++ struct folio_batch *fbatch = NULL; ++ (void)filemap_get_folios_contig(m, &start, ULONG_MAX, fbatch); ++ ],[-Werror]) ++]) ++AC_DEFUN([LC_HAVE_FILEMAP_GET_FOLIOS_CONTIG], [ ++ AC_MSG_CHECKING([if filemap_get_folios_contig() is available]) ++ LB2_LINUX_TEST_RESULT([filemap_get_folios_contig], [ ++ AC_DEFINE(HAVE_FILEMAP_GET_FOLIOS_CONTIG, 1, ++ [filemap_get_folios_contig() is available]) ++ ]) ++]) # LC_HAVE_FILEMAP_GET_FOLIOS_CONTIG ++ + # + # LC_PROG_LINUX + # +@@ -3681,6 +3772,11 @@ AC_DEFUN([LC_PROG_LINUX_SRC], [ + LC_SRC_HAVE_CLASS_CREATE_MODULE_ARG + LC_SRC_HAVE_GET_EXPIRY_TIME64_T + ++ # 6.1 ++ LC_SRC_HAVE_GET_RANDOM_U32_AND_U64 ++ LC_SRC_NFS_FILLDIR_USE_CTX_RETURN_BOOL ++ LC_SRC_HAVE_FILEMAP_GET_FOLIOS_CONTIG ++ + # kernel patch to extend integrity interface + LC_SRC_BIO_INTEGRITY_PREP_FN + ]) +@@ -3920,6 +4016,11 @@ AC_DEFUN([LC_PROG_LINUX_RESULTS], [ + LC_HAVE_CLASS_CREATE_MODULE_ARG + LC_HAVE_GET_EXPIRY_TIME64_T + ++ # 6.1 ++ LC_HAVE_GET_RANDOM_U32_AND_U64 ++ LC_NFS_FILLDIR_USE_CTX_RETURN_BOOL ++ LC_HAVE_FILEMAP_GET_FOLIOS_CONTIG ++ + # kernel patch to extend integrity interface + LC_BIO_INTEGRITY_PREP_FN + ]) +diff --git a/lustre/llite/llite_mmap.c b/lustre/llite/llite_mmap.c +index 1485a29166..16d73ebd71 100644 +--- a/lustre/llite/llite_mmap.c ++++ b/lustre/llite/llite_mmap.c +@@ -51,18 +51,32 @@ void policy_from_vma(union ldlm_policy_data *policy, struct vm_area_struct *vma, + ~PAGE_MASK; + } + ++/* ++ * Linux commit v6.0-rc3-225-gf39af05949a4 ++ * mm: add VMA iterator ++ */ ++#ifndef VMA_ITERATOR ++#define vma_iterator vm_area_struct * ++#define vma_iter_init(vmip, mm, addr) *(vmip) = find_vma(mm, addr) ++#define for_each_vma(vmi, vma) \ ++ for (vma = vmi; vma != NULL; vma = vma->vm_next) ++#endif ++ + struct vm_area_struct *our_vma(struct mm_struct *mm, unsigned long addr, +- size_t count) ++ size_t count) + { + struct vm_area_struct *vma, *ret = NULL; ++ struct vma_iterator vmi; ++ + ENTRY; + + /* mmap_lock must have been held by caller. */ + LASSERT(!mmap_write_trylock(mm)); + +- for (vma = find_vma(mm, addr); +- vma != NULL && vma->vm_start < (addr + count); +- vma = vma->vm_next) { ++ vma_iter_init(&vmi, mm, addr); ++ for_each_vma(vmi, vma) { ++ if (vma->vm_start < (addr + count)) ++ break; + if (vma->vm_ops && vma->vm_ops == &ll_file_vm_ops && + vma->vm_flags & VM_SHARED) { + ret = vma; +diff --git a/lustre/llite/llite_nfs.c b/lustre/llite/llite_nfs.c +index 644aac2456..7d16d9d165 100644 +--- a/lustre/llite/llite_nfs.c ++++ b/lustre/llite/llite_nfs.c +@@ -198,19 +198,10 @@ static int ll_encode_fh(struct inode *inode, u32 *fh, int *plen, + RETURN(FILEID_LUSTRE); + } + +-static int +-#ifndef HAVE_FILLDIR_USE_CTX +-ll_nfs_get_name_filldir(void *cookie, const char *name, int namelen, +- loff_t hash, u64 ino, unsigned type) ++static inline int ++do_nfs_get_name_filldir(struct ll_getname_data *lgd, const char *name, ++ int namelen, loff_t hash, u64 ino, unsigned int type) + { +- struct ll_getname_data *lgd = cookie; +-#else +-ll_nfs_get_name_filldir(struct dir_context *ctx, const char *name, int namelen, +- loff_t hash, u64 ino, unsigned type) +-{ +- struct ll_getname_data *lgd = +- container_of(ctx, struct ll_getname_data, ctx); +-#endif /* HAVE_FILLDIR_USE_CTX */ + /* + * It is hack to access lde_fid for comparison with lgd_fid. + * So the input 'name' must be part of the 'lu_dirent', and +@@ -232,11 +223,41 @@ ll_nfs_get_name_filldir(struct dir_context *ctx, const char *name, int namelen, + lgd->lgd_name[namelen] = 0; + lgd->lgd_found = 1; + } +- return lgd->lgd_found; ++ return lgd->lgd_found; ++} ++ ++#ifdef HAVE_FILLDIR_USE_CTX_RETURN_BOOL ++static bool ++ll_nfs_get_name_filldir(struct dir_context *ctx, const char *name, int namelen, ++ loff_t hash, u64 ino, unsigned int type) ++{ ++ struct ll_getname_data *lgd = ++ container_of(ctx, struct ll_getname_data, ctx); ++ int err = do_nfs_get_name_filldir(lgd, name, namelen, hash, ino, type); ++ ++ return err == 0; ++} ++#elif defined(HAVE_FILLDIR_USE_CTX) ++static int ++ll_nfs_get_name_filldir(struct dir_context *ctx, const char *name, int namelen, ++ loff_t hash, u64 ino, unsigned int type) ++{ ++ struct ll_getname_data *lgd = ++ container_of(ctx, struct ll_getname_data, ctx); ++ ++ return do_nfs_get_name_filldir(lgd, name, namelen, hash, ino, type); + } ++#else ++static int ll_nfs_get_name_filldir(void *cookie, const char *name, int namelen, ++ loff_t hash, u64 ino, unsigned int type) ++{ ++ struct ll_getname_data *lgd = cookie; ++ ++ return do_nfs_get_name_filldir(lgd, name, namelen, hash, ino, type); ++} ++#endif /* HAVE_FILLDIR_USE_CTX */ + +-static int ll_get_name(struct dentry *dentry, char *name, +- struct dentry *child) ++static int ll_get_name(struct dentry *dentry, char *name, struct dentry *child) + { + struct inode *dir = dentry->d_inode; + struct ll_getname_data lgd = { +diff --git a/lustre/llite/vvp_dev.c b/lustre/llite/vvp_dev.c +index 4d7995f040..d5bb6c18e2 100644 +--- a/lustre/llite/vvp_dev.c ++++ b/lustre/llite/vvp_dev.c +@@ -386,6 +386,28 @@ struct vvp_seq_private { + loff_t vvp_prev_pos; + }; + ++unsigned int ll_filemap_get_one_page_contig(struct address_space *mapping, ++ pgoff_t start, struct page **pg) ++{ ++#ifdef HAVE_FILEMAP_GET_FOLIOS_CONTIG ++ struct folio_batch fbatch; ++ int nr; ++ ++ folio_batch_init(&fbatch); ++ *pg = NULL; ++ ++ nr = filemap_get_folios_contig(mapping, &start, start, &fbatch); ++ if (nr == PAGEVEC_SIZE) { ++ --nr; ++ *pg = folio_page(fbatch.folios[nr], 0); ++ return 1; ++ } ++ return 0; ++#else /* !HAVE_FILEMAP_GET_FOLIOS_CONTIG */ ++ return find_get_pages_contig(mapping, start, 1, pg); ++#endif ++} ++ + static struct page *vvp_pgcache_current(struct vvp_seq_private *priv) + { + struct lu_device *dev = &priv->vsp_sbi->ll_cl->cd_lu_dev; +@@ -416,8 +438,9 @@ static struct page *vvp_pgcache_current(struct vvp_seq_private *priv) + } + + inode = vvp_object_inode(priv->vsp_clob); +- nr = find_get_pages_contig(inode->i_mapping, +- priv->vsp_page_index, 1, &vmpage); ++ nr = ll_filemap_get_one_page_contig(inode->i_mapping, ++ priv->vsp_page_index, ++ &vmpage); + if (nr > 0) { + priv->vsp_page_index = vmpage->index; + break; +diff --git a/lustre/obdclass/llog_test.c b/lustre/obdclass/llog_test.c +index e6d36346f8..13d8531fa6 100644 +--- a/lustre/obdclass/llog_test.c ++++ b/lustre/obdclass/llog_test.c +@@ -2249,7 +2249,7 @@ static int llog_test_setup(struct obd_device *obd, struct lustre_cfg *lcfg) + ctxt->loc_dir = o; + llog_ctxt_put(ctxt); + +- llog_test_rand = prandom_u32(); ++ llog_test_rand = get_random_u32(); + + rc = llog_run_tests(&env, tgt); + if (rc) +diff --git a/lustre/obdclass/lu_object.c b/lustre/obdclass/lu_object.c +index 3a07035841..c581211098 100644 +--- a/lustre/obdclass/lu_object.c ++++ b/lustre/obdclass/lu_object.c +@@ -1135,7 +1135,7 @@ int lu_site_init(struct lu_site *s, struct lu_device *top) + return -ENOMEM; + } + +- s->ls_bkt_seed = prandom_u32(); ++ s->ls_bkt_seed = get_random_u32(); + s->ls_bkt_cnt = max_t(long, 1 << LU_SITE_BKT_BITS, + 2 * num_possible_cpus()); + s->ls_bkt_cnt = roundup_pow_of_two(s->ls_bkt_cnt); +diff --git a/lustre/obdclass/lu_tgt_descs.c b/lustre/obdclass/lu_tgt_descs.c +index 1effd99684..02476e0ec6 100644 +--- a/lustre/obdclass/lu_tgt_descs.c ++++ b/lustre/obdclass/lu_tgt_descs.c +@@ -57,7 +57,9 @@ u64 lu_prandom_u64_max(u64 ep_ro) + u64 rand = 0; + + if (ep_ro) { +-#if BITS_PER_LONG == 32 ++#ifdef HAVE_GET_RANDOM_U32_AND_U64 ++ rand = get_random_u64() % ep_ro; ++#elif BITS_PER_LONG == 32 + /* + * If ep_ro > 32-bit, first generate the high + * 32 bits of the random number, then add in the low +@@ -69,9 +71,9 @@ u64 lu_prandom_u64_max(u64 ep_ro) + if (rand == (ep_ro & 0xffffffff00000000ULL)) + rand |= prandom_u32_max((u32)ep_ro); + else +- rand |= prandom_u32(); ++ rand |= get_random_u32(); + #else +- rand = ((u64)prandom_u32() << 32 | prandom_u32()) % ep_ro; ++ rand = ((u64)get_random_u32() << 32 | get_random_u32()) % ep_ro; + #endif + } + +diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c +index d14b22374d..97a9a530a5 100644 +--- a/lustre/osd-ldiskfs/osd_handler.c ++++ b/lustre/osd-ldiskfs/osd_handler.c +@@ -950,7 +950,7 @@ struct osd_check_lmv_buf { + * \retval -ve for failure + */ + #ifdef HAVE_FILLDIR_USE_CTX +-static int osd_stripe_dir_filldir(struct dir_context *buf, ++static FILLDIR_TYPE do_osd_stripe_dir_filldir(struct dir_context *buf, + #else + static int osd_stripe_dir_filldir(void *buf, + #endif +@@ -990,6 +990,8 @@ static int osd_stripe_dir_filldir(void *buf, + return 1; + } + ++WRAP_FILLDIR_FN(do_, osd_stripe_dir_filldir) ++ + /* + * When lookup item under striped directory, we need to locate the master + * MDT-object of the striped directory firstly, then the client will send +@@ -6955,7 +6957,7 @@ struct osd_filldir_cbs { + * \retval 1 on buffer full + */ + #ifdef HAVE_FILLDIR_USE_CTX +-static int osd_ldiskfs_filldir(struct dir_context *ctx, ++static FILLDIR_TYPE do_osd_ldiskfs_filldir(struct dir_context *ctx, + #else + static int osd_ldiskfs_filldir(void *ctx, + #endif +@@ -7027,6 +7029,8 @@ static int osd_ldiskfs_filldir(void *ctx, + RETURN(0); + } + ++WRAP_FILLDIR_FN(do_, osd_ldiskfs_filldir) ++ + /** + * Calls ->iterate*() to load a directory entry at a time + * and stored it in iterator's in-memory data structure. +diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h +index 96de28d311..f7b164ed94 100644 +--- a/lustre/osd-ldiskfs/osd_internal.h ++++ b/lustre/osd-ldiskfs/osd_internal.h +@@ -1701,6 +1701,24 @@ static inline const char *blk_integrity_name(struct blk_integrity *bi) + #define INTEGRITY_FLAG_WRITE BLK_INTEGRITY_GENERATE + #endif + ++#ifdef HAVE_FILLDIR_USE_CTX_RETURN_BOOL ++#define WRAP_FILLDIR_FN(prefix, fill_fn) \ ++static bool fill_fn(struct dir_context *buf, const char *name, int namelen, \ ++ loff_t offset, __u64 ino, unsigned int d_type) \ ++{ \ ++ return !prefix##fill_fn(buf, name, namelen, offset, ino, d_type); \ ++} ++#elif defined(HAVE_FILLDIR_USE_CTX) ++#define WRAP_FILLDIR_FN(prefix, fill_fn) \ ++static int fill_fn(struct dir_context *buf, const char *name, int namelen, \ ++ loff_t offset, __u64 ino, unsigned int d_type) \ ++{ \ ++ return prefix##fill_fn(buf, name, namelen, offset, ino, d_type); \ ++} ++#else ++#define WRAP_FILLDIR_FN(prefix, fill_fn) ++#endif ++ + static inline bool bdev_integrity_enabled(struct block_device *bdev, int rw) + { + #if IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) +diff --git a/lustre/osd-ldiskfs/osd_scrub.c b/lustre/osd-ldiskfs/osd_scrub.c +index ffe4a0b38f..f589fe0fba 100644 +--- a/lustre/osd-ldiskfs/osd_scrub.c ++++ b/lustre/osd-ldiskfs/osd_scrub.c +@@ -1234,27 +1234,30 @@ typedef int (*scandir_t)(struct osd_thread_info *, struct osd_device *, + struct dentry *, filldir_t filldir); + + #ifdef HAVE_FILLDIR_USE_CTX +-static int osd_ios_varfid_fill(struct dir_context *buf, const char *name, +- int namelen, loff_t offset, __u64 ino, +- unsigned d_type); +-static int osd_ios_lf_fill(struct dir_context *buf, const char *name, +- int namelen, loff_t offset, __u64 ino, +- unsigned d_type); +-static int osd_ios_dl_fill(struct dir_context *buf, const char *name, +- int namelen, loff_t offset, __u64 ino, +- unsigned d_type); +-static int osd_ios_uld_fill(struct dir_context *buf, const char *name, +- int namelen, loff_t offset, __u64 ino, +- unsigned d_type); ++static FILLDIR_TYPE ++osd_ios_varfid_fill(struct dir_context *buf, const char *name, int namelen, ++ loff_t offset, __u64 ino, unsigned int d_type); ++ ++static FILLDIR_TYPE ++osd_ios_lf_fill(struct dir_context *buf, const char *name, int namelen, ++ loff_t offset, __u64 ino, unsigned int d_type); ++ ++static FILLDIR_TYPE ++osd_ios_dl_fill(struct dir_context *buf, const char *name, int namelen, ++ loff_t offset, __u64 ino, unsigned int d_type); ++ ++static FILLDIR_TYPE ++osd_ios_uld_fill(struct dir_context *buf, const char *name, int namelen, ++ loff_t offset, __u64 ino, unsigned int d_type); + #else + static int osd_ios_varfid_fill(void *buf, const char *name, int namelen, +- loff_t offset, __u64 ino, unsigned d_type); ++ loff_t offset, __u64 ino, unsigned int d_type); + static int osd_ios_lf_fill(void *buf, const char *name, int namelen, +- loff_t offset, __u64 ino, unsigned d_type); ++ loff_t offset, __u64 ino, unsigned int d_type); + static int osd_ios_dl_fill(void *buf, const char *name, int namelen, +- loff_t offset, __u64 ino, unsigned d_type); ++ loff_t offset, __u64 ino, unsigned int d_type); + static int osd_ios_uld_fill(void *buf, const char *name, int namelen, +- loff_t offset, __u64 ino, unsigned d_type); ++ loff_t offset, __u64 ino, unsigned int d_type); + #endif + + static int +@@ -1883,12 +1886,12 @@ osd_ios_scan_one(struct osd_thread_info *info, struct osd_device *dev, + * or filter_fid_18_23), move them back to its proper /O//d. + */ + #ifdef HAVE_FILLDIR_USE_CTX +-static int osd_ios_lf_fill(struct dir_context *buf, ++static FILLDIR_TYPE do_osd_ios_lf_fill(struct dir_context *buf, + #else + static int osd_ios_lf_fill(void *buf, + #endif + const char *name, int namelen, +- loff_t offset, __u64 ino, unsigned d_type) ++ loff_t offset, __u64 ino, unsigned int d_type) + { + struct osd_ios_filldir_buf *fill_buf = + (struct osd_ios_filldir_buf *)buf; +@@ -1963,14 +1966,15 @@ put: + /* skip the failure to make the scanning to continue. */ + return 0; + } ++WRAP_FILLDIR_FN(do_, osd_ios_lf_fill) + + #ifdef HAVE_FILLDIR_USE_CTX +-static int osd_ios_varfid_fill(struct dir_context *buf, ++static FILLDIR_TYPE do_osd_ios_varfid_fill(struct dir_context *buf, + #else + static int osd_ios_varfid_fill(void *buf, + #endif + const char *name, int namelen, +- loff_t offset, __u64 ino, unsigned d_type) ++ loff_t offset, __u64 ino, unsigned int d_type) + { + struct osd_ios_filldir_buf *fill_buf = + (struct osd_ios_filldir_buf *)buf; +@@ -1999,14 +2003,15 @@ static int osd_ios_varfid_fill(void *buf, + + RETURN(rc); + } ++WRAP_FILLDIR_FN(do_, osd_ios_varfid_fill) + + #ifdef HAVE_FILLDIR_USE_CTX +-static int osd_ios_dl_fill(struct dir_context *buf, ++static FILLDIR_TYPE do_osd_ios_dl_fill(struct dir_context *buf, + #else + static int osd_ios_dl_fill(void *buf, + #endif + const char *name, int namelen, +- loff_t offset, __u64 ino, unsigned d_type) ++ loff_t offset, __u64 ino, unsigned int d_type) + { + struct osd_ios_filldir_buf *fill_buf = + (struct osd_ios_filldir_buf *)buf; +@@ -2044,14 +2049,15 @@ static int osd_ios_dl_fill(void *buf, + + RETURN(rc); + } ++WRAP_FILLDIR_FN(do_, osd_ios_dl_fill) + + #ifdef HAVE_FILLDIR_USE_CTX +-static int osd_ios_uld_fill(struct dir_context *buf, ++static FILLDIR_TYPE do_osd_ios_uld_fill(struct dir_context *buf, + #else + static int osd_ios_uld_fill(void *buf, + #endif + const char *name, int namelen, +- loff_t offset, __u64 ino, unsigned d_type) ++ loff_t offset, __u64 ino, unsigned int d_type) + { + struct osd_ios_filldir_buf *fill_buf = + (struct osd_ios_filldir_buf *)buf; +@@ -2083,14 +2089,15 @@ static int osd_ios_uld_fill(void *buf, + + RETURN(rc); + } ++WRAP_FILLDIR_FN(do_, osd_ios_uld_fill) + + #ifdef HAVE_FILLDIR_USE_CTX +-static int osd_ios_root_fill(struct dir_context *buf, ++static FILLDIR_TYPE do_osd_ios_root_fill(struct dir_context *buf, + #else + static int osd_ios_root_fill(void *buf, + #endif + const char *name, int namelen, +- loff_t offset, __u64 ino, unsigned d_type) ++ loff_t offset, __u64 ino, unsigned int d_type) + { + struct osd_ios_filldir_buf *fill_buf = + (struct osd_ios_filldir_buf *)buf; +@@ -2136,6 +2143,8 @@ out_put: + RETURN(rc); + } + ++WRAP_FILLDIR_FN(do_, osd_ios_root_fill) ++ + static int + osd_ios_general_scan(struct osd_thread_info *info, struct osd_device *dev, + struct dentry *dentry, filldir_t filldir) +-- +2.33.0 + diff --git a/0019-LU-10680-tests-fix-interop-for-sanity-test_160h.patch b/0019-LU-10680-tests-fix-interop-for-sanity-test_160h.patch new file mode 100644 index 0000000000000000000000000000000000000000..22c23ded5a8a616229e7b270c60dbd62bbeed55b --- /dev/null +++ b/0019-LU-10680-tests-fix-interop-for-sanity-test_160h.patch @@ -0,0 +1,54 @@ +From c929f30fd103f9544ee15237f8680839664fdef7 Mon Sep 17 00:00:00 2001 +From: Xing Huang +Date: Thu, 27 Oct 2022 19:41:11 +0800 +Subject: [PATCH 19/49] LU-10680 tests: fix interop for sanity test_160h + +Add a check sanity test_160h whether /sbin/umount.lustre is installed +on the MDS, since this subtest is checking whether the MDS unmount +process has completed, and otherwise fails during interop testing. + +Test-Parameters: testlist=sanity env=ONLY=160 serverversion=2.12 +Fixes: 31fef6845e8b ("LU-10680 mdd: create gc thread when no current transaction") +Signed-off-by: Xing Huang +Change-Id: I6720b9e27a3a92e543ed877453802d23c0eef36d +Reviewed-by: Andreas Dilger +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53773 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + lustre/tests/sanity.sh | 11 +++++++++-- + 1 file changed, 9 insertions(+), 2 deletions(-) + +diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh +index 41959033e8..ca0ea72ade 100755 +--- a/lustre/tests/sanity.sh ++++ b/lustre/tests/sanity.sh +@@ -16522,6 +16522,12 @@ test_160h() { + stop_pids[mds$i]=$! + done + ++ do_facet mds1 "ls -l /sbin/umount*" ++ local umount="umount.lustre" ++ [[ -n "$(do_facet mds1 /bin/ls /sbin/$umount 2>/dev/null)" ]] || ++ umount="umount" ++ echo "using /sbin/$umount on mds1" ++ + for i in $(mdts_nodes); do + local facet + local nb=0 +@@ -16541,8 +16547,9 @@ test_160h() { + "R" 20 || + error "$i: GC-thread not found in R-state" + # check umounts of each MDT on MDS have reached kthread_stop() +- [[ $(do_node $i pgrep umount | wc -l) -eq $nb ]] || +- error "$i: expected $nb umount" ++ do_node $i "ps auxww | grep umount" ++ [[ $(do_node $i pgrep $umount | wc -l) -eq $nb ]] || ++ error "$i: expected $nb $umount" + wait_update $i \ + "ps -C umount -o state --no-headers | uniq" "D" 20 || + error "$i: umount not found in D-state" +-- +2.33.0 + diff --git a/0019-LU-14668-lnet-add-lock_prim_nid-lnet-module-paramete.patch b/0019-LU-14668-lnet-add-lock_prim_nid-lnet-module-paramete.patch deleted file mode 100644 index 5b41c683d88b92f89cda9652ef901087717a0db4..0000000000000000000000000000000000000000 --- a/0019-LU-14668-lnet-add-lock_prim_nid-lnet-module-paramete.patch +++ /dev/null @@ -1,245 +0,0 @@ -From 6cfc8e55a2e77c9c91b81a8842e2cbd886025298 Mon Sep 17 00:00:00 2001 -From: Serguei Smirnov -Date: Tue, 28 Feb 2023 15:02:20 -0800 -Subject: [PATCH 19/61] LU-14668 lnet: add 'lock_prim_nid" lnet module - parameter - -Add 'lock_prim_nid' lnet module parameter to allow control -of how Lustre peer primary NID is selected. -If set to 1 (default), the NID specified by Lustre when -calling LNet API is designated as primary for the peer, -allowing for non-blocking discovery in the background. -If set to 0, peer discovery is blocking until complete -and the NID listed first in discovery response is designated -as primary. - -Lustre-change: https://review.whamcloud.com/50159 -Lustre-commit: fc7a0d6013b46ebc17cdfdccc04a5d1d92c6af24 - -Signed-off-by: Serguei Smirnov -Change-Id: I6ed1cb0c637f4aa7a7340a6f01819ba9a85858f4 -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51134 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Frank Sehr -Reviewed-by: Cyril Bordage -Reviewed-by: Oleg Drokin ---- - lnet/include/lnet/lib-lnet.h | 1 + - lnet/lnet/api-ni.c | 5 ++ - lnet/lnet/peer.c | 106 +++++++++++++++++++++++------------ - 3 files changed, 75 insertions(+), 37 deletions(-) - -diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h -index 5805586fb2..5821d20be2 100644 ---- a/lnet/include/lnet/lib-lnet.h -+++ b/lnet/include/lnet/lib-lnet.h -@@ -615,6 +615,7 @@ extern int alive_router_check_interval; - extern int live_router_check_interval; - extern int dead_router_check_interval; - extern int portal_rotor; -+extern int lock_prim_nid; - - void lnet_mt_event_handler(struct lnet_event *event); - -diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c -index 3a56ea83a1..fa7d702c2f 100644 ---- a/lnet/lnet/api-ni.c -+++ b/lnet/lnet/api-ni.c -@@ -243,6 +243,11 @@ module_param_call(lnet_response_tracking, response_tracking_set, param_get_int, - MODULE_PARM_DESC(lnet_response_tracking, - "(0|1|2|3) LNet Internal Only|GET Reply only|PUT ACK only|Full Tracking (default)"); - -+int lock_prim_nid = 1; -+module_param(lock_prim_nid, int, 0444); -+MODULE_PARM_DESC(lock_prim_nid, -+ "Whether nid passed down by Lustre is locked as primary"); -+ - #define LNET_LND_TIMEOUT_DEFAULT ((LNET_TRANSACTION_TIMEOUT_DEFAULT - 1) / \ - (LNET_RETRY_COUNT_DEFAULT + 1)) - unsigned int lnet_lnd_timeout = LNET_LND_TIMEOUT_DEFAULT; -diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c -index 33a16f8470..4eb7a44cf4 100644 ---- a/lnet/lnet/peer.c -+++ b/lnet/lnet/peer.c -@@ -1411,6 +1411,7 @@ LNetAddPeer(lnet_nid_t *nids, __u32 num_nids) - lnet_nid_t pnid = 0; - bool mr; - int i, rc; -+ int flags = lock_prim_nid ? LNET_PEER_LOCK_PRIMARY : 0; - - if (!nids || num_nids < 1) - return -EINVAL; -@@ -1431,7 +1432,7 @@ LNetAddPeer(lnet_nid_t *nids, __u32 num_nids) - if (!pnid) { - pnid = nids[i]; - rc = lnet_add_peer_ni(pnid, LNET_NID_ANY, mr, -- LNET_PEER_LOCK_PRIMARY); -+ flags); - if (rc == -EALREADY) { - struct lnet_peer *lp; - -@@ -1447,10 +1448,10 @@ LNetAddPeer(lnet_nid_t *nids, __u32 num_nids) - } - } else if (lnet_peer_discovery_disabled) { - rc = lnet_add_peer_ni(nids[i], LNET_NID_ANY, mr, -- LNET_PEER_LOCK_PRIMARY); -+ flags); - } else { - rc = lnet_add_peer_ni(pnid, nids[i], mr, -- LNET_PEER_LOCK_PRIMARY); -+ flags); - } - - if (rc && rc != -EEXIST) -@@ -1493,37 +1494,54 @@ LNetPrimaryNID(lnet_nid_t nid) - * down then this discovery can introduce long delays into the mount - * process, so skip it if it isn't necessary. - */ -+again: - spin_lock(&lp->lp_lock); -- if (!lnet_peer_discovery_disabled && -- (!(lp->lp_state & LNET_PEER_LOCK_PRIMARY) || -- !lnet_peer_is_uptodate_locked(lp))) { -- /* force a full discovery cycle */ -- lp->lp_state |= LNET_PEER_FORCE_PING | LNET_PEER_FORCE_PUSH | -- LNET_PEER_LOCK_PRIMARY; -+ if (!(lp->lp_state & LNET_PEER_LOCK_PRIMARY) && lock_prim_nid) -+ lp->lp_state |= LNET_PEER_LOCK_PRIMARY; -+ -+ /* DD disabled, nothing to do */ -+ if (lnet_peer_discovery_disabled) { -+ nid = lnet_nid_to_nid4(&lp->lp_primary_nid); - spin_unlock(&lp->lp_lock); -+ goto out_decref; -+ } - -- /* start discovery in the background. Messages to that -- * peer will not go through until the discovery is -- * complete -- */ -- rc = lnet_discover_peer_locked(lpni, cpt, false); -- if (rc) -- goto out_decref; -- /* The lpni (or lp) for this NID may have changed and our ref is -- * the only thing keeping the old one around. Release the ref -- * and lookup the lpni again -- */ -- lnet_peer_ni_decref_locked(lpni); -- lpni = lnet_find_peer_ni_locked(nid); -- if (!lpni) { -- rc = -ENOENT; -- goto out_unlock; -- } -- lp = lpni->lpni_peer_net->lpn_peer; -- } else { -+ /* Peer already up to date, nothing to do */ -+ if (lnet_peer_is_uptodate_locked(lp)) { -+ nid = lnet_nid_to_nid4(&lp->lp_primary_nid); - spin_unlock(&lp->lp_lock); -+ goto out_decref; -+ } -+ spin_unlock(&lp->lp_lock); -+ -+ /* If primary nid locking is enabled, discovery is performed -+ * in the background. -+ * If primary nid locking is disabled, discovery blocks here. -+ * Messages to the peer will not go through until the discovery is -+ * complete. -+ */ -+ if (lock_prim_nid) -+ rc = lnet_discover_peer_locked(lpni, cpt, false); -+ else -+ rc = lnet_discover_peer_locked(lpni, cpt, true); -+ if (rc) -+ goto out_decref; -+ -+ /* The lpni (or lp) for this NID may have changed and our ref is -+ * the only thing keeping the old one around. Release the ref -+ * and lookup the lpni again -+ */ -+ lnet_peer_ni_decref_locked(lpni); -+ lpni = lnet_find_peer_ni_locked(nid); -+ if (!lpni) { -+ rc = -ENOENT; -+ goto out_unlock; - } -- primary_nid = lnet_nid_to_nid4(&lp->lp_primary_nid); -+ lp = lpni->lpni_peer_net->lpn_peer; -+ -+ if (!lock_prim_nid && !lnet_is_discovery_disabled(lp)) -+ goto again; -+ nid = lnet_nid_to_nid4(&lp->lp_primary_nid); - out_decref: - lnet_peer_ni_decref_locked(lpni); - out_unlock: -@@ -1614,7 +1632,6 @@ lnet_peer_attach_peer_ni(struct lnet_peer *lp, - ptable->pt_peers++; - } - -- - /* Update peer state */ - spin_lock(&lp->lp_lock); - if (flags & LNET_PEER_CONFIGURED) { -@@ -1692,10 +1709,8 @@ lnet_peer_add(lnet_nid_t nid4, unsigned int flags) - rc = -EPERM; - goto out; - } else if (lp->lp_state & LNET_PEER_LOCK_PRIMARY) { -- if (nid_same(&lp->lp_primary_nid, &nid)) { -+ if (nid_same(&lp->lp_primary_nid, &nid)) - rc = -EEXIST; -- goto out; -- } - /* we're trying to recreate an existing peer which - * has already been created and its primary - * locked. This is likely due to two servers -@@ -1703,8 +1718,19 @@ lnet_peer_add(lnet_nid_t nid4, unsigned int flags) - * to that node with the primary NID which was - * first added by Lustre - */ -- rc = -EALREADY; -+ else -+ rc = -EALREADY; - goto out; -+ } else if (!(flags & -+ (LNET_PEER_LOCK_PRIMARY | LNET_PEER_CONFIGURED))) { -+ /* if not recreating peer as configured and -+ * not locking primary nid, no need to -+ * do anything if primary nid is not being changed -+ */ -+ if (nid_same(&lp->lp_primary_nid, &nid)) { -+ rc = -EEXIST; -+ goto out; -+ } - } - /* Delete and recreate the peer. - * We can get here: -@@ -2011,6 +2037,14 @@ __must_hold(&the_lnet.ln_api_mutex) - lnet_peer_ni_decref_locked(lpni); - lp = lpni->lpni_peer_net->lpn_peer; - -+ /* Peer must have been configured. */ -+ if ((flags & LNET_PEER_CONFIGURED) && -+ !(lp->lp_state & LNET_PEER_CONFIGURED)) { -+ CDEBUG(D_NET, "peer %s was not configured\n", -+ libcfs_nid2str(prim_nid)); -+ return -ENOENT; -+ } -+ - /* Primary NID must match */ - if (lnet_nid_to_nid4(&lp->lp_primary_nid) != prim_nid) { - CDEBUG(D_NET, "prim_nid %s is not primary for peer %s\n", -@@ -2026,9 +2060,7 @@ __must_hold(&the_lnet.ln_api_mutex) - return -EPERM; - } - -- if ((flags & LNET_PEER_LOCK_PRIMARY) && -- (lnet_peer_is_uptodate(lp) && -- (lp->lp_state & LNET_PEER_LOCK_PRIMARY))) { -+ if (lnet_peer_is_uptodate(lp) && !(flags & LNET_PEER_CONFIGURED)) { - CDEBUG(D_NET, - "Don't add temporary peer NI for uptodate peer %s\n", - libcfs_nidstr(&lp->lp_primary_nid)); --- -2.33.0 - diff --git a/0020-LU-14668-tests-verify-state-of-peer-added-with-lock_.patch b/0020-LU-14668-tests-verify-state-of-peer-added-with-lock_.patch deleted file mode 100644 index 9a4646b8a6256075fc707ff123422a567b772c1c..0000000000000000000000000000000000000000 --- a/0020-LU-14668-tests-verify-state-of-peer-added-with-lock_.patch +++ /dev/null @@ -1,48 +0,0 @@ -From 7ee579d25a614946ba22a5a08fdc4373c41ef8f1 Mon Sep 17 00:00:00 2001 -From: Serguei Smirnov -Date: Thu, 9 Mar 2023 15:00:46 -0800 -Subject: [PATCH 20/61] LU-14668 tests: verify state of peer added with - '--lock_prim' - -Add peer state verification to sanity-lnet test_26: -check that peer state has corresponding bit set for a peer -created with '--lock_prim' option. - -Lustre-change: https://review.whamcloud.com/50249 -Lustre-commit: 9b6fcfa334b153e52caec16d4cfd180306826a3a - -Test-Parameters: trivial testlist=sanity-lnet -Fixes: 05f7f6a0b ("LU-14668 lnet: add 'force' option to lnetctl peer del") -Signed-off-by: Serguei Smirnov -Change-Id: Id5fde036907f9dd19a21e8e6611a070321310f0e -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51135 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Frank Sehr -Reviewed-by: Cyril Bordage -Reviewed-by: Oleg Drokin ---- - lustre/tests/sanity-lnet.sh | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh -index 4af5e96ef8..1fb298776c 100755 ---- a/lustre/tests/sanity-lnet.sh -+++ b/lustre/tests/sanity-lnet.sh -@@ -937,6 +937,13 @@ test_26() { - - do_lnetctl peer add --prim_nid 1.1.1.1@tcp --lock_prim || - error "Peer add with --lock_prim option failed $?" -+ local peer_state=$($LNETCTL peer show -v 4 --nid 1.1.1.1@tcp | -+ awk '/peer state/ {print $NF}') -+ # This relies on the following peer state definition: -+ # #define LNET_PEER_LOCK_PRIMARY BIT(20) -+ if ((!("$peer_state" & (1 << 20)))); then -+ error "Peer state does not have 'locked' bit set: $peer_state" -+ fi - do_lnetctl peer del --prim_nid 1.1.1.1@tcp || - error "Peer del failed $?" - $LNETCTL peer show --nid 1.1.1.1@tcp | grep -q 1.1.1.1@tcp || --- -2.33.0 - diff --git a/0020-LU-17409-scripts-correct-ldev-MGS-handling.patch b/0020-LU-17409-scripts-correct-ldev-MGS-handling.patch new file mode 100644 index 0000000000000000000000000000000000000000..f3675a4562e2465ccf358cdb21f8c3aa1e048396 --- /dev/null +++ b/0020-LU-17409-scripts-correct-ldev-MGS-handling.patch @@ -0,0 +1,151 @@ +From d6e31fd32d693ca5f0dbc8ef79c7cca95ba9990b Mon Sep 17 00:00:00 2001 +From: "Olaf P. Faaland" +Date: Mon, 8 Jan 2024 21:36:50 -0800 +Subject: [PATCH 20/49] LU-17409 scripts: correct ldev MGS handling + +ldev was incorrectly parsing the line specifying the hosts that can +run the MGS, when of the form + + gopher1 gopher2 MGS gopher1/mgs + +as it assumed every target specified included a filesystem name, +like 'lustre3-MDT0000'. + +This corrects that, assuming that an MGS may not be related to a +specific file system. + +When such an input line is found, assume that MGS is used by any file +systems included in the ldev.conf. When user includes option '-F +' as well as '-R MGS', include that MGS, in the targets +reported. + +Lustre-change: https://review.whamcloud.com/53619 +Lustre-commit: bb6a2d2e80f04645b488ecca6ba14cb628e3eeb3 + +Test-Parameters: trivial +Signed-off-by: Olaf Faaland +Change-Id: Ifab5db1dfb094755e29747ec6b90d1566b16c18c +Reviewed-by: Cameron Harr +Reviewed-by: Eric Carbonneau +Reviewed-by: Andreas Dilger +Reviewed-by: Gian-Carlo DeFazio +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54667 +Reviewed-by: Oleg Drokin +Tested-by: jenkins +Tested-by: Maloo +--- + lustre/scripts/ldev | 48 +++++++++++++++++++++++++++++++++------------ + 1 file changed, 36 insertions(+), 12 deletions(-) + +diff --git a/lustre/scripts/ldev b/lustre/scripts/ldev +index 77c0040bdf..55d24185f5 100755 +--- a/lustre/scripts/ldev ++++ b/lustre/scripts/ldev +@@ -46,6 +46,8 @@ my %eparse = ( + + my %conf = (); + ++my $global_mgs = ""; ++ + # + # Main + # +@@ -159,6 +161,7 @@ sub parse_config + $type = $1; + $dev = $2; + } ++ + eparse_line ($line, "efieldcount") if (!defined $dev); + eparse_line ($line, "epairwise") if (exists $l2f{$local} + && $l2f{$local} ne $foreign); +@@ -166,25 +169,38 @@ sub parse_config + + eparse_line ($line, "elabel_uniq") if (exists $label2dev{$label} + || exists $label2local{$label}); +- /(\w+)-(OST|MDT|MGS)([0-9a-fA-F]{4})/, $label; +- my $filesys = $1; +- my $nodetype = $2; ++ + $label2dev{$label} = $dev; + $label2local{$label} = $local; + $label2journal{$label} = $j if defined $j; + $label2raidtab{$label} = $raidtab if defined $raidtab; ++ $label2hostname{$label}=$local; + if (defined $type) { + $label2type{$label} = $type; + if ($type eq "zfs" && $dev =~ m{^([^/]+)/[^/]+$}) { + $label2zpool{$label} = $1; + } + } +- $label2hostname{$label}=$local; + +- if ($nodetype eq "MGS") { ++ my $filesys; ++ my $nodetype; ++ if ($label eq "MGS") { ++ $filesys = ""; ++ $nodetype = $label; ++ } else { ++ /(\w+)-(OST|MDT|MGS)([0-9a-fA-F]{4})/, $label; ++ $filesys = $1; ++ $nodetype = $2; ++ } ++ ++ if ($nodetype eq "MGS" or $label eq "MGS") { + $filesys2mgs{$filesys} = $label; + } + ++ if ($label eq "MGS") { ++ $global_mgs = $label; ++ } ++ + next if $conf{role} and lc $conf{role} ne lc $nodetype; + + if ($local eq $conf{hostname}) { +@@ -456,6 +472,7 @@ sub exec_cmd + @labels = (@{$conf{local_labels}}, @{$conf{foreign_labels}}); + } elsif ($conf{fsname}) { + @labels = (@labels, @{$conf{fs_labels}}); ++ push(@labels, $global_mgs) if ($global_mgs); + } else { + @labels = @{$conf{local_labels}}; + } +@@ -481,21 +498,28 @@ sub exec_cmd + my $hostname = $label2hostname{$_}; + my $mgsnid; + if ($cmd =~ /%m/) { +- if (!exists $filesys2mgs{$fsname}) { ++ my $mgs; ++ my $mgs_host; ++ ++ if (exists $filesys2mgs{$fsname}) { ++ $mgs = $filesys2mgs{$fsname}; ++ } elsif ($global_mgs) { ++ $mgs = $global_mgs; ++ } else { + log_fatal ("$fsname has no MGS defined\n"); + } + +- my $mgs = $filesys2mgs{$fsname}; +- if (!exists $label2hostname{$mgs}) { ++ if (exists $label2hostname{$mgs}) { ++ $mgs_host = $label2hostname{$mgs}; ++ } else { + log_fatal ("$mgs has no hostname defined\n"); + } + + my %host2nid = %{$conf{host2nid}}; +- $mgs = $label2hostname{$mgs}; +- if (!exists $host2nid{$mgs}) { +- log_fatal ("$mgs has no NID defined\n"); ++ if (!exists $host2nid{$mgs_host}) { ++ log_fatal ("$mgs and $mgs_host have no NID defined\n"); + } +- $mgsnid = $host2nid{$mgs}; ++ $mgsnid = $host2nid{$mgs_host}; + } + + $cmd =~ s/%f/$fsname/g; # %f = fsname +-- +2.33.0 + diff --git a/0021-LU-11787-test-Fix-checkfilemap-tests-for-64K-page.patch b/0021-LU-11787-test-Fix-checkfilemap-tests-for-64K-page.patch deleted file mode 100644 index 0197a03a34887ba38cf8a1639d14b1311546a949..0000000000000000000000000000000000000000 --- a/0021-LU-11787-test-Fix-checkfilemap-tests-for-64K-page.patch +++ /dev/null @@ -1,93 +0,0 @@ -From 66d9916dc947064b598f52476fb6482c0bbaff10 Mon Sep 17 00:00:00 2001 -From: James Simmons -Date: Mon, 31 Jan 2022 12:44:46 -0500 -Subject: [PATCH 21/61] LU-11787 test: Fix checkfilemap tests for 64K page - -File mapping is page size aligned. Modify the tests to handle 64K -page. - -Lustre-change: https://review.whamcloud.com/45629 -Lustre-commit: 7c88dfd28b5cc6114a85f187ecb2473657d42c9d - -Test-Parameters: trivial clientdistro=el8.7 clientarch=aarch64 testlist=sanityn env=ONLY="71a 71b" -Change-Id: I316a197db8cdd0f9064431f8c572b43adf6110b8 -Signed-off-by: James Simmons -Signed-off-by: Xinliang Liu -Reviewed-by: Arshad Hussain -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51287 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/tests/sanityn.sh | 22 ++++++++-------------- - 1 file changed, 8 insertions(+), 14 deletions(-) - -diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh -index 17a1231d4c..ca366fd10a 100755 ---- a/lustre/tests/sanityn.sh -+++ b/lustre/tests/sanityn.sh -@@ -22,12 +22,6 @@ ALWAYS_EXCEPT="$SANITYN_EXCEPT " - ALWAYS_EXCEPT+=" 28 " - # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! - --# skip tests for PPC until they are fixed --if [[ $(uname -m) = ppc64 ]]; then -- # bug number: LU-11787 -- ALWAYS_EXCEPT+=" 71a" --fi -- - if [ $mds1_FSTYPE = "zfs" ]; then - # bug number: LU-15757 (test_102() causes crash in umount later) - ALWAYS_EXCEPT+=" 102" -@@ -3560,28 +3554,28 @@ test_71a() { - checkfiemap --test || - skip "checkfiemap not runnable: $?" - # write data this way: hole - data - hole - data -- dd if=/dev/urandom of=$DIR1/$tfile bs=40K seek=1 count=1 -+ dd if=/dev/urandom of=$DIR1/$tfile bs=64K seek=1 count=1 - [ "$(facet_fstype ost$(($($LFS getstripe -i $DIR1/$tfile) + 1)))" = \ - "zfs" ] && - skip "ORI-366/LU-1941: FIEMAP unimplemented on ZFS" && return 0 -- dd if=/dev/urandom of=$DIR1/$tfile bs=40K seek=3 count=1 -+ dd if=/dev/urandom of=$DIR1/$tfile bs=64K seek=3 count=1 - GET_STAT="lctl get_param -n ldlm.services.ldlm_cbd.stats" - stat $DIR2/$tfile - local can1=$($GET_STAT | awk '/ldlm_bl_callback/ {print $2}') - echo $can1 -- checkfiemap $DIR2/$tfile 81920 || -+ checkfiemap $DIR2/$tfile 131072 || - error "data is not flushed from client" - local can2=$($GET_STAT | awk '/ldlm_bl_callback/ {print $2}') - echo $can2 - - # common case of "create file, copy file" on a single node - # should not flush data from ost -- dd if=/dev/urandom of=$DIR1/$tfile bs=40K seek=1 count=1 -- dd if=/dev/urandom of=$DIR1/$tfile bs=40K seek=3 count=1 -+ dd if=/dev/urandom of=$DIR1/$tfile bs=64K seek=1 count=1 -+ dd if=/dev/urandom of=$DIR1/$tfile bs=64K seek=3 count=1 - stat $DIR1/$tfile - local can3=$($GET_STAT | awk '/ldlm_bl_callback/ {print $2}') - echo $can3 -- checkfiemap $DIR1/$tfile 81920 || -+ checkfiemap $DIR1/$tfile 131072 || - error 4 - local can4=$($GET_STAT | awk '/ldlm_bl_callback/ {print $2}') - echo $can2 -@@ -3605,11 +3599,11 @@ test_71b() { - mkdir -p $DIR1/$tdir - - $LFS setstripe -c -1 $DIR1/$tdir || error "setstripe failed" -- dd if=/dev/urandom of=$DIR1/$tdir/$tfile bs=40K count=1 -+ dd if=/dev/urandom of=$DIR1/$tdir/$tfile bs=64K count=1 - [ "$(facet_fstype ost$(($($LFS getstripe -i $DIR1/$tdir/$tfile) + 1)))" = \ - "zfs" ] && - skip "ORI-366/LU-1941: FIEMAP unimplemented on ZFS" && return 0 -- checkfiemap $DIR1/$tdir/$tfile 40960 || error "checkfiemap failed" -+ checkfiemap $DIR1/$tdir/$tfile 65536 || error "checkfiemap failed" - } - run_test 71b "check fiemap support for stripecount > 1" - --- -2.33.0 - diff --git a/0021-LU-15947-obdclass-improve-precision-of-wakeups-for-m.patch b/0021-LU-15947-obdclass-improve-precision-of-wakeups-for-m.patch new file mode 100644 index 0000000000000000000000000000000000000000..afc38f2756bcfdece18b422bc072ce55251e86b5 --- /dev/null +++ b/0021-LU-15947-obdclass-improve-precision-of-wakeups-for-m.patch @@ -0,0 +1,301 @@ +From a622bfcf3b1c3530199fb740735ee70592a85032 Mon Sep 17 00:00:00 2001 +From: Mr NeilBrown +Date: Mon, 21 Jun 2021 13:25:42 +1000 +Subject: [PATCH 21/49] LU-15947 obdclass: improve precision of wakeups for + mod_rpcs + +There is a limit of the number of in-flight mod rpcs with a +complication that a 'close' rpc is always permitted if there are no +other close rpcs in flight, even if that would exceed the limit. + +When a non-close-request complete, we just wake the first waiting +request and assume it will use the slot we released. When a +close-request completes, the first waiting request may not find a slot +if the close was using the 'extra' slot. So in that case we wake all +waiting requests and let them fit it out. This is wasteful and +unfair. + +To correct this we revise the wait/wake approach to use a dedicated +wakeup function which atomically checks if a given task can proceed, +and updates the counters when permission to proceed is given. This +means that once a task has been woken, it has already been accounted +and it can proceed. + +To minimise locking, cl_mod_rpcs_lock is discarded and +cl_mod_rpcs_waitq.lock is used to protect the counters. For the +fast-path where the max has not been reached, this means we take and +release that spinlock just once. We call wake_up_locked while still +holding the lock, and if that woke the process, then we don't drop the +spinlock to wait, but proceed directly to the remainder of the task. + +When the last 'close' rpc completes, the wake function will iterate +the whole wait queue until it finds a task waiting to submit a close +request. When any other rpc completes, the queue will only be +searched until the maximum is reached. + +Lustre-change: https://review.whamcloud.com/44041 +Lustre-commit: 5243630b09d22e0b576d81390d604774881f63f7 + +Signed-off-by: Mr NeilBrown +Change-Id: Iff094c3188a3bd8a04edc1d5d98ec3014e2b059b +Reviewed-by: James Simmons +Reviewed-by: Petros Koutoupis +Signed-off-by: Etienne AUJAMES +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51538 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + lustre/include/obd.h | 1 - + lustre/ldlm/ldlm_lib.c | 1 - + lustre/obdclass/genops.c | 149 +++++++++++++++++++++------------------ + 3 files changed, 82 insertions(+), 69 deletions(-) + +diff --git a/lustre/include/obd.h b/lustre/include/obd.h +index 86f13a143a..36ead2093b 100644 +--- a/lustre/include/obd.h ++++ b/lustre/include/obd.h +@@ -330,7 +330,6 @@ struct client_obd { + + /* modify rpcs in flight + * currently used for metadata only */ +- spinlock_t cl_mod_rpcs_lock; + __u16 cl_max_mod_rpcs_in_flight; + __u16 cl_mod_rpcs_in_flight; + __u16 cl_close_rpcs_in_flight; +diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c +index 1d7a29d76f..887b42b99b 100644 +--- a/lustre/ldlm/ldlm_lib.c ++++ b/lustre/ldlm/ldlm_lib.c +@@ -506,7 +506,6 @@ int client_obd_setup(struct obd_device *obd, struct lustre_cfg *lcfg) + cli->cl_max_rpcs_in_flight = OBD_MAX_RIF_DEFAULT; + } + +- spin_lock_init(&cli->cl_mod_rpcs_lock); + spin_lock_init(&cli->cl_mod_rpcs_hist.oh_lock); + cli->cl_max_mod_rpcs_in_flight = 0; + cli->cl_mod_rpcs_in_flight = 0; +diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c +index 787a67c85b..969bcc5162 100644 +--- a/lustre/obdclass/genops.c ++++ b/lustre/obdclass/genops.c +@@ -2176,16 +2176,16 @@ int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max) + return -ERANGE; + } + +- spin_lock(&cli->cl_mod_rpcs_lock); ++ spin_lock_irq(&cli->cl_mod_rpcs_waitq.lock); + + prev = cli->cl_max_mod_rpcs_in_flight; + cli->cl_max_mod_rpcs_in_flight = max; + + /* wakeup waiters if limit has been increased */ + if (cli->cl_max_mod_rpcs_in_flight > prev) +- wake_up(&cli->cl_mod_rpcs_waitq); ++ wake_up_locked(&cli->cl_mod_rpcs_waitq); + +- spin_unlock(&cli->cl_mod_rpcs_lock); ++ spin_unlock_irq(&cli->cl_mod_rpcs_waitq.lock); + + return 0; + } +@@ -2197,7 +2197,7 @@ int obd_mod_rpc_stats_seq_show(struct client_obd *cli, + unsigned long mod_tot = 0, mod_cum; + int i; + +- spin_lock(&cli->cl_mod_rpcs_lock); ++ spin_lock_irq(&cli->cl_mod_rpcs_waitq.lock); + lprocfs_stats_header(seq, ktime_get_real(), cli->cl_mod_rpcs_init, 25, + ":", true, ""); + seq_printf(seq, "modify_RPCs_in_flight: %hu\n", +@@ -2220,7 +2220,7 @@ int obd_mod_rpc_stats_seq_show(struct client_obd *cli, + break; + } + +- spin_unlock(&cli->cl_mod_rpcs_lock); ++ spin_unlock_irq(&cli->cl_mod_rpcs_waitq.lock); + + return 0; + } +@@ -2235,10 +2235,26 @@ EXPORT_SYMBOL(obd_mod_rpc_stats_seq_show); + * On the MDC client, to avoid a potential deadlock (see Bugzilla 3462), + * one close request is allowed above the maximum. + */ +-static inline bool obd_mod_rpc_slot_avail_locked(struct client_obd *cli, +- bool close_req) ++struct mod_waiter { ++ struct client_obd *cli; ++ bool close_req; ++ wait_queue_entry_t wqe; ++}; ++static int claim_mod_rpc_function(wait_queue_entry_t *wq_entry, ++ unsigned int mode, int flags, void *key) + { ++ struct mod_waiter *w = container_of(wq_entry, struct mod_waiter, wqe); ++ struct client_obd *cli = w->cli; ++ bool close_req = w->close_req; + bool avail; ++ int ret; ++ ++ /* As woken_wake_function() doesn't remove us from the wait_queue, ++ * we could get called twice for the same thread - take care. ++ */ ++ if (wq_entry->flags & WQ_FLAG_WOKEN) ++ /* Already woke this thread, don't try again */ ++ return 0; + + /* A slot is available if + * - number of modify RPCs in flight is less than the max +@@ -2246,22 +2262,25 @@ static inline bool obd_mod_rpc_slot_avail_locked(struct client_obd *cli, + */ + avail = cli->cl_mod_rpcs_in_flight < cli->cl_max_mod_rpcs_in_flight || + (close_req && cli->cl_close_rpcs_in_flight == 0); +- +- return avail; +-} +- +-static inline bool obd_mod_rpc_slot_avail(struct client_obd *cli, +- bool close_req) +-{ +- bool avail; +- +- spin_lock(&cli->cl_mod_rpcs_lock); +- avail = obd_mod_rpc_slot_avail_locked(cli, close_req); +- spin_unlock(&cli->cl_mod_rpcs_lock); +- return avail; ++ if (avail) { ++ cli->cl_mod_rpcs_in_flight++; ++ if (w->close_req) ++ cli->cl_close_rpcs_in_flight++; ++ ret = woken_wake_function(wq_entry, mode, flags, key); ++ } else if (cli->cl_close_rpcs_in_flight) ++ /* No other waiter could be woken */ ++ ret = -1; ++ else if (key == NULL) ++ /* This was not a wakeup from a close completion, so there is no ++ * point seeing if there are close waiters to be woken ++ */ ++ ret = -1; ++ else ++ /* There might be be a close we could wake, keep looking */ ++ ret = 0; ++ return ret; + } + +- + /* Get a modify RPC slot from the obd client @cli according + * to the kind of operation @opc that is going to be sent + * and the intent @it of the operation if it applies. +@@ -2272,47 +2291,47 @@ static inline bool obd_mod_rpc_slot_avail(struct client_obd *cli, + */ + __u16 obd_get_mod_rpc_slot(struct client_obd *cli, __u32 opc) + { +- bool close_req = false; ++ struct mod_waiter wait = { ++ .cli = cli, ++ .close_req = (opc == MDS_CLOSE), ++ }; + __u16 i, max; + +- if (opc == MDS_CLOSE) +- close_req = true; +- +- do { +- spin_lock(&cli->cl_mod_rpcs_lock); +- max = cli->cl_max_mod_rpcs_in_flight; +- if (obd_mod_rpc_slot_avail_locked(cli, close_req)) { +- /* there is a slot available */ +- cli->cl_mod_rpcs_in_flight++; +- if (close_req) +- cli->cl_close_rpcs_in_flight++; +- lprocfs_oh_tally(&cli->cl_mod_rpcs_hist, +- cli->cl_mod_rpcs_in_flight); +- /* find a free tag */ +- i = find_first_zero_bit(cli->cl_mod_tag_bitmap, +- max + 1); +- LASSERT(i < OBD_MAX_RIF_MAX); +- LASSERT(!test_and_set_bit(i, cli->cl_mod_tag_bitmap)); +- spin_unlock(&cli->cl_mod_rpcs_lock); +- /* tag 0 is reserved for non-modify RPCs */ +- +- CDEBUG(D_RPCTRACE, +- "%s: modify RPC slot %u is allocated opc %u, max %hu\n", +- cli->cl_import->imp_obd->obd_name, +- i + 1, opc, max); +- +- return i + 1; +- } +- spin_unlock(&cli->cl_mod_rpcs_lock); +- +- CDEBUG(D_RPCTRACE, "%s: sleeping for a modify RPC slot " +- "opc %u, max %hu\n", +- cli->cl_import->imp_obd->obd_name, opc, max); ++ init_wait(&wait.wqe); ++ wait.wqe.func = claim_mod_rpc_function; + +- wait_event_idle_exclusive(cli->cl_mod_rpcs_waitq, +- obd_mod_rpc_slot_avail(cli, +- close_req)); +- } while (true); ++ spin_lock_irq(&cli->cl_mod_rpcs_waitq.lock); ++ __add_wait_queue(&cli->cl_mod_rpcs_waitq, &wait.wqe); ++ /* This wakeup will only succeed if the maximums haven't ++ * been reached. If that happens, WQ_FLAG_WOKEN will be cleared ++ * and there will be no need to wait. ++ */ ++ wake_up_locked(&cli->cl_mod_rpcs_waitq); ++ if (!(wait.wqe.flags & WQ_FLAG_WOKEN)) { ++ spin_unlock_irq(&cli->cl_mod_rpcs_waitq.lock); ++ wait_woken(&wait.wqe, TASK_UNINTERRUPTIBLE, ++ MAX_SCHEDULE_TIMEOUT); ++ spin_lock_irq(&cli->cl_mod_rpcs_waitq.lock); ++ } ++ __remove_wait_queue(&cli->cl_mod_rpcs_waitq, &wait.wqe); ++ ++ max = cli->cl_max_mod_rpcs_in_flight; ++ lprocfs_oh_tally(&cli->cl_mod_rpcs_hist, ++ cli->cl_mod_rpcs_in_flight); ++ /* find a free tag */ ++ i = find_first_zero_bit(cli->cl_mod_tag_bitmap, ++ max + 1); ++ LASSERT(i < OBD_MAX_RIF_MAX); ++ LASSERT(!test_and_set_bit(i, cli->cl_mod_tag_bitmap)); ++ spin_unlock_irq(&cli->cl_mod_rpcs_waitq.lock); ++ /* tag 0 is reserved for non-modify RPCs */ ++ ++ CDEBUG(D_RPCTRACE, ++ "%s: modify RPC slot %u is allocated opc %u, max %hu\n", ++ cli->cl_import->imp_obd->obd_name, ++ i + 1, opc, max); ++ ++ return i + 1; + } + EXPORT_SYMBOL(obd_get_mod_rpc_slot); + +@@ -2329,19 +2348,15 @@ void obd_put_mod_rpc_slot(struct client_obd *cli, __u32 opc, __u16 tag) + if (opc == MDS_CLOSE) + close_req = true; + +- spin_lock(&cli->cl_mod_rpcs_lock); ++ spin_lock_irq(&cli->cl_mod_rpcs_waitq.lock); + cli->cl_mod_rpcs_in_flight--; + if (close_req) + cli->cl_close_rpcs_in_flight--; + /* release the tag in the bitmap */ + LASSERT(tag - 1 < OBD_MAX_RIF_MAX); + LASSERT(test_and_clear_bit(tag - 1, cli->cl_mod_tag_bitmap) != 0); +- spin_unlock(&cli->cl_mod_rpcs_lock); +- /* LU-14741 - to prevent close RPCs stuck behind normal ones */ +- if (close_req) +- wake_up_all(&cli->cl_mod_rpcs_waitq); +- else +- wake_up(&cli->cl_mod_rpcs_waitq); ++ __wake_up_locked_key(&cli->cl_mod_rpcs_waitq, TASK_NORMAL, ++ (void *)close_req); ++ spin_unlock_irq(&cli->cl_mod_rpcs_waitq.lock); + } + EXPORT_SYMBOL(obd_put_mod_rpc_slot); +- +-- +2.33.0 + diff --git a/0022-LU-15800-ofd-take-a-read-lock-for-fallocate.patch b/0022-LU-15800-ofd-take-a-read-lock-for-fallocate.patch deleted file mode 100644 index 42a83dff35b0a8eb6717677774f6aa8ad0d07c00..0000000000000000000000000000000000000000 --- a/0022-LU-15800-ofd-take-a-read-lock-for-fallocate.patch +++ /dev/null @@ -1,51 +0,0 @@ -From 8299b3fd77ebcc372b5d929eaa08231fc703c431 Mon Sep 17 00:00:00 2001 -From: Alex Zhuravlev -Date: Tue, 10 May 2022 10:48:55 +0300 -Subject: [PATCH 22/61] LU-15800 ofd: take a read lock for fallocate - -there is no need to take an write (exclusive) object's -lock for fallocate - we just need to serialize fallocate -vs destroy, all internal structures should be protected -by OSD and disk filesystem like the write path does. - -Lustre-change: https://review.whamcloud.com/47268 -Lustre-commit: 5fae80066162ea637c8649f6439fc14e1d9a7cf8 - -Fixes: cdaaa87f6b ("LU-14214 ofd: fix locking in ofd_object_fallocate()") -Signed-off-by: Alex Zhuravlev -Change-Id: I65986745865ee329c5257a7efca5e79403830608 -Reviewed-by: Arshad Hussain -Reviewed-by: Andreas Dilger -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51702 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/ofd/ofd_objects.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/lustre/ofd/ofd_objects.c b/lustre/ofd/ofd_objects.c -index 41fe74bb11..16f9fba3d8 100644 ---- a/lustre/ofd/ofd_objects.c -+++ b/lustre/ofd/ofd_objects.c -@@ -801,7 +801,7 @@ int ofd_object_fallocate(const struct lu_env *env, struct ofd_object *fo, - if (rc) - GOTO(stop, rc); - -- ofd_write_lock(env, fo); -+ ofd_read_lock(env, fo); - if (!ofd_object_exists(fo)) - GOTO(unlock, rc = -ENOENT); - -@@ -824,7 +824,7 @@ int ofd_object_fallocate(const struct lu_env *env, struct ofd_object *fo, - filter_fid_le_to_cpu(&fo->ofo_ff, ff, sizeof(*ff)); - } - unlock: -- ofd_write_unlock(env, fo); -+ ofd_read_unlock(env, fo); - stop: - ofd_trans_stop(env, ofd, th, rc); - RETURN(rc); --- -2.33.0 - diff --git a/0022-LU-17510-obdclass-fix-wake-up-when-queuing-close-req.patch b/0022-LU-17510-obdclass-fix-wake-up-when-queuing-close-req.patch new file mode 100644 index 0000000000000000000000000000000000000000..876d2b42a0cfac51315229155cbb5980a09fc77b --- /dev/null +++ b/0022-LU-17510-obdclass-fix-wake-up-when-queuing-close-req.patch @@ -0,0 +1,102 @@ +From 1732704711488d2d233f0b8e5bc9814f443405c6 Mon Sep 17 00:00:00 2001 +From: Mr NeilBrown +Date: Mon, 4 Mar 2024 13:15:17 +1100 +Subject: [PATCH 22/49] LU-17510 obdclass: fix wake up when queuing close + request. + +The waitqueue for requests that need to be sent but that haven't been +allocated a slot is kept ordered by request arrival for fairness. So +new requests are added to the end. + +For requests other than 'close' there is a limit to the number of +active requests (slots) and requests are assigned to slot on a +first-come-first-served basis, so they are simply removed from the +head of the list. + +For 'close' requests it is important that these not block indefinitely +behind other other requests so there is one slot that can only be used +by a close request - and only if no other slots are used by a close +request. These requests do not follow a strict FIFO order. + +When a non-"close" request completes we wake the first request on the +list. There is no point searching all the way down the list for a +close request that could also be woken. We only do that when a +"close" request completes. This optimises the common case. + +However: when a request is first queued we add it to the end of the +queue and then wake up the first deserving request if there is one. +When there are free slots, this is expected to wake the request just +queued. When there are no free slots, nothing is woken. + +When a "close" request is queued and added to the end of the queue +after other non-close requests, we need to potentially search to the +end of the queue for a close request to wake, just as we do when a +close request completes. Unfortunately we don't. This can result in +a close request blocking indefinitely. + +So: change the wakeup in obd_get_mod_rpc_slot() to match the wakeup in +obd_put_mod_rpc_slot(). This ensure consistent handling and in +particular will handle a close request immediately if there are no +other close requests in flight. + +Clarify comment in claim_mod_rpc_function() to make and perform minor +code cleanup there. + +Lustre-change: https://review.whamcloud.com/54259 +Lustre-commit: 7a2296a397381a5f6f9473b297f0062e8ff15948 + +Fixes: b5fde4d6c023 ("LU-17197 obdclass: preserve fairness when waiting for rpc slot") +Signed-off-by: Mr NeilBrown +Change-Id: I7b658efc0298a091166f0f18ce460fc3148047eb +Reviewed-by: James Simmons +Reviewed-by: Shaun Tancheff +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54689 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + lustre/obdclass/genops.c | 15 ++++++++++----- + 1 file changed, 10 insertions(+), 5 deletions(-) + +diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c +index 969bcc5162..8487b0379a 100644 +--- a/lustre/obdclass/genops.c ++++ b/lustre/obdclass/genops.c +@@ -2264,15 +2264,16 @@ static int claim_mod_rpc_function(wait_queue_entry_t *wq_entry, + (close_req && cli->cl_close_rpcs_in_flight == 0); + if (avail) { + cli->cl_mod_rpcs_in_flight++; +- if (w->close_req) ++ if (close_req) + cli->cl_close_rpcs_in_flight++; + ret = woken_wake_function(wq_entry, mode, flags, key); + } else if (cli->cl_close_rpcs_in_flight) + /* No other waiter could be woken */ + ret = -1; +- else if (key == NULL) +- /* This was not a wakeup from a close completion, so there is no +- * point seeing if there are close waiters to be woken ++ else if (!key) ++ /* This was not a wakeup from a close completion or a new close ++ * being queued, so there is no point seeing if there are close ++ * waiters to be woken. + */ + ret = -1; + else +@@ -2305,8 +2306,12 @@ __u16 obd_get_mod_rpc_slot(struct client_obd *cli, __u32 opc) + /* This wakeup will only succeed if the maximums haven't + * been reached. If that happens, WQ_FLAG_WOKEN will be cleared + * and there will be no need to wait. ++ * If a close_req was enqueue, ensure we search all the way to the ++ * end of the waitqueue for a close request. + */ +- wake_up_locked(&cli->cl_mod_rpcs_waitq); ++ __wake_up_locked_key(&cli->cl_mod_rpcs_waitq, TASK_NORMAL, ++ (void*)wait.close_req); ++ + if (!(wait.wqe.flags & WQ_FLAG_WOKEN)) { + spin_unlock_irq(&cli->cl_mod_rpcs_waitq.lock); + wait_woken(&wait.wqe, TASK_UNINTERRUPTIBLE, +-- +2.33.0 + diff --git a/0023-LU-15851-lnet-Adjust-niov-checks-for-large-MD.patch b/0023-LU-15851-lnet-Adjust-niov-checks-for-large-MD.patch new file mode 100644 index 0000000000000000000000000000000000000000..1d3f988a29649fb559858709b345524768b13ab3 --- /dev/null +++ b/0023-LU-15851-lnet-Adjust-niov-checks-for-large-MD.patch @@ -0,0 +1,90 @@ +From 34734b656f49e88851a1fa01b5095c268f70b2fa Mon Sep 17 00:00:00 2001 +From: Chris Horn +Date: Sat, 16 Apr 2022 10:01:57 -0600 +Subject: [PATCH 23/49] LU-15851 lnet: Adjust niov checks for large MD + +An LNet user can allocate a large contiguous MD. That MD can have > +LNET_MAX_IOV pages which causes some LNDs to assert on either niov +argument passed to lnd_recv() or the value stored in +lnet_msg::msg_niov. This is true even in cases where the actual +transfer size is <= LNET_MTU and will not exceed limits in the LNDs. + +Adjust ksocklnd_send()/ksocklnd_recv() to assert on the return value +of lnet_extract_kiov(). + +Remove the assert on msg_niov (payload_niov) from kiblnd_send(). +kiblnd_setup_rd_kiov() will already fail if we exceed ko2iblnd's +available scatter gather entries. + +Lustre-change: https://review.whamcloud.com/47319 +Lustre-commit: 105193b4a147257a0f9332053a16eb676dc99623 + +HPE-bug-id: LUS-10878 +Test-Parameters: trivial +Fixes: 857f11169f ("LU-13004 lnet: always put a page list into struct lnet_libmd") +Signed-off-by: Chris Horn +Change-Id: Iaa851d90f735d04e5167bb9c07235625759245b2 +Reviewed-by: Shaun Tancheff +Reviewed-by: Serguei Smirnov +Reviewed-by: Alexey Lyashkov +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54848 +Tested-by: jenkins +Tested-by: Maloo +Reviewed-by: Oleg Drokin +--- + lnet/klnds/o2iblnd/o2iblnd_cb.c | 1 - + lnet/klnds/socklnd/socklnd_cb.c | 5 +++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c +index 75895d69b0..01d7a6a8a1 100644 +--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c ++++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c +@@ -1674,7 +1674,6 @@ kiblnd_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg) + payload_nob, payload_niov, libcfs_idstr(target)); + + LASSERT (payload_nob == 0 || payload_niov > 0); +- LASSERT (payload_niov <= LNET_MAX_IOV); + + /* Thread context */ + LASSERT (!in_interrupt()); +diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c +index 4e7a61b3f5..be20dba5f4 100644 +--- a/lnet/klnds/socklnd/socklnd_cb.c ++++ b/lnet/klnds/socklnd/socklnd_cb.c +@@ -995,7 +995,6 @@ ksocknal_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg) + payload_nob, payload_niov, libcfs_idstr(target)); + + LASSERT (payload_nob == 0 || payload_niov > 0); +- LASSERT (payload_niov <= LNET_MAX_IOV); + LASSERT (!in_interrupt ()); + + desc_size = offsetof(struct ksock_tx, +@@ -1022,6 +1021,8 @@ ksocknal_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg) + payload_niov, payload_kiov, + payload_offset, payload_nob); + ++ LASSERT(tx->tx_nkiov <= LNET_MAX_IOV); ++ + if (payload_nob >= *ksocknal_tunables.ksnd_zc_min_payload) + tx->tx_zc_capable = 1; + +@@ -1365,7 +1366,6 @@ ksocknal_recv(struct lnet_ni *ni, void *private, struct lnet_msg *msg, + struct ksock_sched *sched = conn->ksnc_scheduler; + + LASSERT (mlen <= rlen); +- LASSERT (niov <= LNET_MAX_IOV); + + conn->ksnc_lnet_msg = msg; + conn->ksnc_rx_nob_wanted = mlen; +@@ -1385,6 +1385,7 @@ ksocknal_recv(struct lnet_ni *ni, void *private, struct lnet_msg *msg, + niov, kiov, offset, mlen); + } + ++ LASSERT(conn->ksnc_rx_nkiov <= LNET_MAX_IOV); + LASSERT (mlen == + lnet_iov_nob (conn->ksnc_rx_niov, conn->ksnc_rx_iov) + + lnet_kiov_nob (conn->ksnc_rx_nkiov, conn->ksnc_rx_kiov)); +-- +2.33.0 + diff --git a/0023-LU-16873-osd-update-OI_Scrub-file-with-new-magic.patch b/0023-LU-16873-osd-update-OI_Scrub-file-with-new-magic.patch deleted file mode 100644 index 18b58437ab774fb197aff042f3a97ba7a91a8963..0000000000000000000000000000000000000000 --- a/0023-LU-16873-osd-update-OI_Scrub-file-with-new-magic.patch +++ /dev/null @@ -1,93 +0,0 @@ -From 703d8a994bf33ac07f8bd4c956880db3d9abb016 Mon Sep 17 00:00:00 2001 -From: Alexander Zarochentsev -Date: Sun, 28 May 2023 08:42:27 -0400 -Subject: [PATCH 23/61] LU-16873 osd: update OI_Scrub file with new magic - -The fix for LUS-11542 detects the format change correctly -but does not write new oi scrub file magic, so new mount -triggers the "oi files counter reset" again and again. - -Lustre-change: https://review.whamcloud.com/51226 -Lustre-commit: 38b7c408212f60d684c9b114d90b4514e0044ffe - -Fixes: 126275ba83 ("LU-16655 scrub: upgrade scrub_file from 2.12 format") -HPE-bug-id: LUS-11646 -Signed-off-by: Alexander Zarochentsev -Change-Id: Ia13fcfaf0d8f2c4ee9331dd9fec0ff159d195186 -Reviewed-by: Andreas Dilger -Reviewed-by: Andrew Perepechko -Signed-off-by: Etienne AUJAMES -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51525 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/obdclass/scrub.c | 2 ++ - lustre/osd-ldiskfs/osd_oi.c | 15 ++++++--------- - 2 files changed, 8 insertions(+), 9 deletions(-) - -diff --git a/lustre/obdclass/scrub.c b/lustre/obdclass/scrub.c -index 89c3f752da..cc100244c0 100644 ---- a/lustre/obdclass/scrub.c -+++ b/lustre/obdclass/scrub.c -@@ -127,6 +127,7 @@ EXPORT_SYMBOL(scrub_file_init); - void scrub_file_reset(struct lustre_scrub *scrub, uuid_t uuid, u64 flags) - { - struct scrub_file *sf = &scrub->os_file; -+ ENTRY; - - CDEBUG(D_LFSCK, "%s: reset OI scrub file, old flags = " - "%#llx, add flags = %#llx\n", -@@ -150,6 +151,7 @@ void scrub_file_reset(struct lustre_scrub *scrub, uuid_t uuid, u64 flags) - sf->sf_items_igif = 0; - if (!scrub->os_in_join) - sf->sf_items_updated_prior = 0; -+ EXIT; - } - EXPORT_SYMBOL(scrub_file_reset); - -diff --git a/lustre/osd-ldiskfs/osd_oi.c b/lustre/osd-ldiskfs/osd_oi.c -index 21468fec15..b361796e41 100644 ---- a/lustre/osd-ldiskfs/osd_oi.c -+++ b/lustre/osd-ldiskfs/osd_oi.c -@@ -316,7 +316,7 @@ osd_oi_table_open(struct osd_thread_info *info, struct osd_device *osd, - - if (rc == -ENOENT && create == false) { - if (oi_count == 0) -- return count; -+ RETURN(count); - - rc = 0; - ldiskfs_set_bit(i, sf->sf_oi_bitmap); -@@ -425,23 +425,20 @@ int osd_oi_init(struct osd_thread_info *info, struct osd_device *osd, - if (count == sf->sf_oi_count) - GOTO(out, rc = count); - -- if (sf->sf_oi_count == 0) { -- if (likely((count & (count - 1)) == 0)) -- GOTO(out, rc = count); -- -- LCONSOLE_WARN( -- "%s: invalid oi count %d, remove them, then set it to %d\n", -+ /* Trust the counted number of OI files if it is sane */ -+ if ((count & (count - 1)) != 0) { -+ LCONSOLE_WARN("%s: invalid oi count %d, remove them, then set it to %d\n", - osd_dev2name(osd), count, osd_oi_count); - osd_oi_table_put(info, oi, count); - rc = osd_remove_ois(info, osd); - if (rc) - GOTO(out, rc); - -- sf->sf_oi_count = osd_oi_count; -+ count = osd_oi_count; - } - - scrub_file_reset(scrub, osd->od_uuid, SF_RECREATED); -- count = sf->sf_oi_count; -+ sf->sf_oi_count = count; - goto create; - } - --- -2.33.0 - diff --git a/0024-LU-15519-quota-fallocate-does-not-increase-projectid.patch b/0024-LU-15519-quota-fallocate-does-not-increase-projectid.patch deleted file mode 100644 index 3d77bca48d74362ec892b7db8eb1d50d6c4328f5..0000000000000000000000000000000000000000 --- a/0024-LU-15519-quota-fallocate-does-not-increase-projectid.patch +++ /dev/null @@ -1,224 +0,0 @@ -From 691387d77bfe9260b65669978fb3f988055fc1e0 Mon Sep 17 00:00:00 2001 -From: Arshad Hussain -Date: Mon, 14 Feb 2022 14:06:47 +0530 -Subject: [PATCH 24/61] LU-15519 quota: fallocate does not increase projectid - usage - -fallocate() was not accounting for projectid quota usage. -This was happening due to two reasons. 1) the projectid -was not properly passed to md_op_data in ll_set_project() -and 2) the OBD_MD_FLPROJID flag was not set receive the -projctid. - -This patch addresses the above reasons. - -Test-case: sanity-quota/78a added - -Lustre-change: https://review.whamcloud.com/46676 -Lustre-commit: 5fc934ebbbe665f24e2f11fe224065dd8e9a08ba - -Fixes: 48457868a02a ("LU-3606 fallocate: Implement fallocate preallocate operation") -Signed-off-by: Arshad Hussain -Change-Id: I3ed44e7ef7ca8fe49a08133449c33b62b1eff500 -Reviewed-by: Andreas Dilger -Reviewed-by: Hongchao Zhang -Signed-off-by: Etienne AUJAMES -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51535 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/include/cl_object.h | 1 + - lustre/llite/file.c | 15 ++++++++---- - lustre/llite/vvp_object.c | 3 ++- - lustre/lov/lov_io.c | 2 ++ - lustre/osc/osc_io.c | 9 ++++--- - lustre/tests/sanity-quota.sh | 47 ++++++++++++++++++++++++++++++++++++ - 6 files changed, 68 insertions(+), 9 deletions(-) - -diff --git a/lustre/include/cl_object.h b/lustre/include/cl_object.h -index e4fb55a705..d44d59b074 100644 ---- a/lustre/include/cl_object.h -+++ b/lustre/include/cl_object.h -@@ -1874,6 +1874,7 @@ struct cl_io { - loff_t sa_falloc_end; - uid_t sa_falloc_uid; - gid_t sa_falloc_gid; -+ __u32 sa_falloc_projid; - } ci_setattr; - struct cl_data_version_io { - u64 dv_data_version; -diff --git a/lustre/llite/file.c b/lustre/llite/file.c -index 219931e06b..71f11a9ea2 100644 ---- a/lustre/llite/file.c -+++ b/lustre/llite/file.c -@@ -2834,7 +2834,7 @@ static int ll_do_fiemap(struct inode *inode, struct fiemap *fiemap, - GOTO(out, rc); - } - -- fmkey.lfik_oa.o_valid = OBD_MD_FLID | OBD_MD_FLGROUP; -+ fmkey.lfik_oa.o_valid = OBD_MD_FLID | OBD_MD_FLGROUP | OBD_MD_FLPROJID; - obdo_from_inode(&fmkey.lfik_oa, inode, OBD_MD_FLSIZE); - obdo_set_parent_fid(&fmkey.lfik_oa, &ll_i2info(inode)->lli_fid); - -@@ -3575,8 +3575,8 @@ int ll_ioctl_check_project(struct inode *inode, __u32 xflags, - - static int ll_set_project(struct inode *inode, __u32 xflags, __u32 projid) - { -- struct md_op_data *op_data; - struct ptlrpc_request *req = NULL; -+ struct md_op_data *op_data; - struct cl_object *obj; - unsigned int inode_flags; - int rc = 0; -@@ -3594,7 +3594,10 @@ static int ll_set_project(struct inode *inode, __u32 xflags, __u32 projid) - op_data->op_attr_flags = ll_inode_to_ext_flags(inode_flags); - if (xflags & FS_XFLAG_PROJINHERIT) - op_data->op_attr_flags |= LUSTRE_PROJINHERIT_FL; -+ -+ /* pass projid to md_op_data */ - op_data->op_projid = projid; -+ - op_data->op_xvalid |= OP_XVALID_PROJID | OP_XVALID_FLAGS; - rc = md_setattr(ll_i2sbi(inode)->ll_md_exp, op_data, NULL, 0, &req); - ptlrpc_req_finished(req); -@@ -5454,11 +5457,11 @@ int ll_getattr(struct vfsmount *mnt, struct dentry *de, struct kstat *stat) - int cl_falloc(struct file *file, struct inode *inode, int mode, loff_t offset, - loff_t len) - { -+ loff_t size = i_size_read(inode); - struct lu_env *env; - struct cl_io *io; - __u16 refcheck; - int rc; -- loff_t size = i_size_read(inode); - - ENTRY; - -@@ -5477,12 +5480,14 @@ int cl_falloc(struct file *file, struct inode *inode, int mode, loff_t offset, - io->u.ci_setattr.sa_falloc_end = offset + len; - io->u.ci_setattr.sa_subtype = CL_SETATTR_FALLOCATE; - -- CDEBUG(D_INODE, "UID %u GID %u\n", -+ CDEBUG(D_INODE, "UID %u GID %u PRJID %u\n", - from_kuid(&init_user_ns, inode->i_uid), -- from_kgid(&init_user_ns, inode->i_gid)); -+ from_kgid(&init_user_ns, inode->i_gid), -+ ll_i2info(inode)->lli_projid); - - io->u.ci_setattr.sa_falloc_uid = from_kuid(&init_user_ns, inode->i_uid); - io->u.ci_setattr.sa_falloc_gid = from_kgid(&init_user_ns, inode->i_gid); -+ io->u.ci_setattr.sa_falloc_projid = ll_i2info(inode)->lli_projid; - - if (io->u.ci_setattr.sa_falloc_end > size) { - loff_t newsize = io->u.ci_setattr.sa_falloc_end; -diff --git a/lustre/llite/vvp_object.c b/lustre/llite/vvp_object.c -index 2413da9498..ab5c68b797 100644 ---- a/lustre/llite/vvp_object.c -+++ b/lustre/llite/vvp_object.c -@@ -198,7 +198,8 @@ static void vvp_req_attr_set(const struct lu_env *env, struct cl_object *obj, - { - struct inode *inode; - struct obdo *oa; -- u64 valid_flags = OBD_MD_FLTYPE | OBD_MD_FLUID | OBD_MD_FLGID; -+ u64 valid_flags = OBD_MD_FLTYPE | OBD_MD_FLUID | OBD_MD_FLGID | -+ OBD_MD_FLPROJID; - - oa = attr->cra_oa; - inode = vvp_object_inode(obj); -diff --git a/lustre/lov/lov_io.c b/lustre/lov/lov_io.c -index ce4fa30b84..381ae56699 100644 ---- a/lustre/lov/lov_io.c -+++ b/lustre/lov/lov_io.c -@@ -694,6 +694,8 @@ static void lov_io_sub_inherit(struct lov_io_sub *sub, struct lov_io *lio, - parent->u.ci_setattr.sa_falloc_uid; - io->u.ci_setattr.sa_falloc_gid = - parent->u.ci_setattr.sa_falloc_gid; -+ io->u.ci_setattr.sa_falloc_projid = -+ parent->u.ci_setattr.sa_falloc_projid; - } - if (cl_io_is_trunc(io)) { - loff_t new_size = parent->u.ci_setattr.sa_attr.lvb_size; -diff --git a/lustre/osc/osc_io.c b/lustre/osc/osc_io.c -index e4bd2738a6..86399655f5 100644 ---- a/lustre/osc/osc_io.c -+++ b/lustre/osc/osc_io.c -@@ -680,11 +680,14 @@ static int osc_io_setattr_start(const struct lu_env *env, - oa->o_blocks = io->u.ci_setattr.sa_falloc_end; - oa->o_uid = io->u.ci_setattr.sa_falloc_uid; - oa->o_gid = io->u.ci_setattr.sa_falloc_gid; -+ oa->o_projid = io->u.ci_setattr.sa_falloc_projid; - oa->o_valid |= OBD_MD_FLSIZE | OBD_MD_FLBLOCKS | -- OBD_MD_FLUID | OBD_MD_FLGID; -+ OBD_MD_FLUID | OBD_MD_FLGID | OBD_MD_FLPROJID; - -- CDEBUG(D_INODE, "size %llu blocks %llu uid %u gid %u\n", -- oa->o_size, oa->o_blocks, oa->o_uid, oa->o_gid); -+ CDEBUG(D_INODE, -+ "size %llu blocks %llu uid %u gid %u prjid %u\n", -+ oa->o_size, oa->o_blocks, oa->o_uid, oa->o_gid, -+ oa->o_projid); - result = osc_fallocate_base(osc_export(cl2osc(obj)), - oa, osc_async_upcall, - cbargs, falloc_mode); -diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh -index 047b1a9687..f36601ce5d 100755 ---- a/lustre/tests/sanity-quota.sh -+++ b/lustre/tests/sanity-quota.sh -@@ -5251,6 +5251,53 @@ test_78() - } - run_test 78 "Check fallocate increase quota usage" - -+test_78a() -+{ -+ (( $CLIENT_VERSION >= $(version_code 2.15.3) )) || -+ skip "need client at least 2.15.3" -+ (( $OST1_VERSION >= $(version_code 2.15.3) )) || -+ skip "need OST at least 2.15.3" -+ check_set_fallocate_or_skip -+ -+ setup_quota_test || error "setup quota failed with $?" -+ -+ # enable ost quota -+ set_ost_qtype $QTYPE || error "enable ost quota failed" -+ -+ mkdir -p $DIR/$tdir || error "failed to create $tdir" -+ -+ local projectid=5200 # Random project id to test -+ -+ change_project -sp $projectid $DIR/$tdir -+ -+ # setup quota limit -+ $LFS setquota -p $projectid -b25M -B25M $DIR/$tdir || -+ error "lfs setquota project failed" -+ -+ # call fallocate -+ fallocate -l 204800 $DIR/$tdir/$tfile -+ -+ # Get curspace (kbytes) for $projectid -+ local kbytes=$(getquota -p $projectid global curspace) -+ -+ echo "kbytes returned:$kbytes" -+ -+ # For file size of 204800. We should be having roughly 200 kbytes -+ # returned. Anything alarmingly low (50 taken as arbitrary value) -+ # would bail out this TC. Also this also avoids $kbytes of 0 -+ # to be used in calculation below. -+ (( $kbytes > 50 )) || -+ error "fallocate did not use projectid. kbytes returned:$kbytes" -+ -+ local expect_lo=$(($kbytes * 95 / 100)) # 5% below -+ local expect_hi=$(($kbytes * 105 / 100)) # 5% above -+ -+ # Verify kbytes is 200 (204800/1024). With a permited 5% drift -+ (( $kbytes >= $expect_lo && $kbytes <= $expect_hi )) || -+ error "fallocate did not use quota projectid correctly" -+} -+run_test 78a "Check fallocate increase projectid usage" -+ - test_79() - { - local qpool="qpool1" --- -2.33.0 - diff --git a/0024-LU-17010-lfsck-don-t-dump-stack-repeatedly.patch b/0024-LU-17010-lfsck-don-t-dump-stack-repeatedly.patch new file mode 100644 index 0000000000000000000000000000000000000000..ecfa538facc34e37cd10d880d02a7e9d73fcb872 --- /dev/null +++ b/0024-LU-17010-lfsck-don-t-dump-stack-repeatedly.patch @@ -0,0 +1,48 @@ +From 1d54befc9bc69188092dc4ef18d301c43b9ca8cd Mon Sep 17 00:00:00 2001 +From: Andreas Dilger +Date: Tue, 12 Sep 2023 23:12:18 -0600 +Subject: [PATCH 24/49] LU-17010 lfsck: don't dump stack repeatedly + +If there are transactions started with LFSCK in dry-run mode, don't +dump the stack repeatedly, as this can spam the console logs and +significantly hurt performance. + +Lustre-commit: dc360cd3eff20618f243ab89097a62f8ecf2c929 +Lustre-change: https://review.whamcloud.com/52356 + +Test-Parameters: trivial testlist=sanity-lfsck +Fixes: 0c1ae1cb9c ("LU-13124 scrub: check for multiple linked file") +Signed-off-by: Andreas Dilger +Change-Id: I0b0d64911453dc8ab947e284656311b5d0300c1e +Reviewed-by: Hongchao Zhang +Reviewed-by: Lai Siyao +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54853 +Reviewed-by: Oleg Drokin +Tested-by: jenkins +Tested-by: Maloo +--- + lustre/lfsck/lfsck_internal.h | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/lustre/lfsck/lfsck_internal.h b/lustre/lfsck/lfsck_internal.h +index e6a8a48ae7..8e048e706c 100644 +--- a/lustre/lfsck/lfsck_internal.h ++++ b/lustre/lfsck/lfsck_internal.h +@@ -1549,10 +1549,13 @@ lfsck_trans_create(const struct lu_env *env, struct dt_device *dev, + struct lfsck_instance *lfsck) + { + if (lfsck->li_bookmark_ram.lb_param & LPF_DRYRUN) { ++ static int count; ++ + CERROR("%s: transaction is being created in DRYRUN mode!\n", + lfsck_lfsck2name(lfsck)); + +- dump_stack(); ++ if (count++ < 3) ++ dump_stack(); + return ERR_PTR(-EINVAL); + } + +-- +2.33.0 + diff --git a/0025-LU-16060-osd-ldiskfs-copy-nul-byte-terminator-in-wri.patch b/0025-LU-16060-osd-ldiskfs-copy-nul-byte-terminator-in-wri.patch deleted file mode 100644 index 68197cb5eec84ac5e6f2a20a481cc10fefebf5ed..0000000000000000000000000000000000000000 --- a/0025-LU-16060-osd-ldiskfs-copy-nul-byte-terminator-in-wri.patch +++ /dev/null @@ -1,47 +0,0 @@ -From d1f0ef129d8e9af13c31dc821a00639de3349873 Mon Sep 17 00:00:00 2001 -From: Alexander Zarochentsev -Date: Wed, 20 Jul 2022 19:05:53 +0300 -Subject: [PATCH 25/61] LU-16060 osd-ldiskfs: copy nul byte terminator in - writelink - -memcpy() call in osd_ldiskfs_writelink() doesn't copy the nul -terminator byte from the source buffer, leaving the space -after target link name uninialized which is ok for the kernel -code and debugfs but not e2fsck. - -HPE-bug-id: LUS-11103 - -Lustre-change: https://review.whamcloud.com/48092 -Lustre-commit: 907dc0a2d333f2df2d654a968fc50f8cc05b779d - -Signed-off-by: Alexander Zarochentsev -Change-Id: I914f2c78e1a6571bf360a23b0ede8c70502bf0df -Reviewed-by: Artem Blagodarenko -Reviewed-by: Andrew Perepechko -Reviewed-by: Andreas Dilger -Signed-off-by: Etienne AUJAMES -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51356 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/osd-ldiskfs/osd_io.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c -index f44420dda2..7131add58d 100644 ---- a/lustre/osd-ldiskfs/osd_io.c -+++ b/lustre/osd-ldiskfs/osd_io.c -@@ -2022,7 +2022,8 @@ static int osd_ldiskfs_writelink(struct inode *inode, char *buffer, int buflen) - /* LU-2634: clear the extent format for fast symlink */ - ldiskfs_clear_inode_flag(inode, LDISKFS_INODE_EXTENTS); - -- memcpy((char *)&LDISKFS_I(inode)->i_data, (char *)buffer, buflen); -+ /* Copying the NUL byte terminating the link target as well */ -+ memcpy((char *)&LDISKFS_I(inode)->i_data, (char *)buffer, buflen + 1); - spin_lock(&inode->i_lock); - LDISKFS_I(inode)->i_disksize = buflen; - i_size_write(inode, buflen); --- -2.33.0 - diff --git a/0025-LU-16502-lutf-fix-bugs-in-bash-scripts.patch b/0025-LU-16502-lutf-fix-bugs-in-bash-scripts.patch new file mode 100644 index 0000000000000000000000000000000000000000..a0670a5041b2e9237b4935831d43a2044df6806e --- /dev/null +++ b/0025-LU-16502-lutf-fix-bugs-in-bash-scripts.patch @@ -0,0 +1,115 @@ +From e1684d9ff51574d9da9b45a0affd611e0767c49a Mon Sep 17 00:00:00 2001 +From: Timothy Day +Date: Thu, 21 Mar 2024 12:06:51 -0700 +Subject: [PATCH 25/49] LU-16502 lutf: fix bugs in bash scripts + +Addressed some issues I found when running +LUTF. The "rm" fails without a file to remove. +A Lustre wiki led me to source the script, but this +will log out of the current shell. Adding a warning +against doing this. + +Also, fix shellcheck errors and a few warnings in +LUTF related scripts. Added bash shebangs, since +shellcheck requires these to lint the scripts. + +Lustre-change: https://review.whamcloud.com/49728 +Lustre-commit: 3cd0bb6968fc3432b7d1641bf0489e4e4b1e809f + +Test-Parameters: trivial env=SANITY_EXCEPT="101j" testlist=sanity +Signed-off-by: Timothy Day +Change-Id: I501d58d25bfcd6564755485b9a1afa2277848b96 +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54519 +Tested-by: Maloo +Tested-by: jenkins +Reviewed-by: Oleg Drokin +--- + lustre/tests/lutf.sh | 23 ++++++++++++------- + lustre/tests/lutf/python/config/lutf_start.sh | 3 ++- + 2 files changed, 17 insertions(+), 9 deletions(-) + +diff --git a/lustre/tests/lutf.sh b/lustre/tests/lutf.sh +index f512e9ef30..64f1b9ac1a 100644 +--- a/lustre/tests/lutf.sh ++++ b/lustre/tests/lutf.sh +@@ -1,7 +1,14 @@ ++#!/bin/bash + # Run select tests by setting ONLY, or as arguments to the script. + # Skip specific tests by setting EXCEPT. + # + ++# Sourcing this script will log you out from your shell ++if [[ -n "${PS1}" ]]; then ++ echo "Do not source this script. Run using ./lutf.sh instead." ++ return 1 ++fi ++ + export ONLY=${ONLY:-"$*"} + export SUITE=${SUITE:-"$*"} + export PATTERN=${PATTERN:-"$*"} +@@ -9,11 +16,11 @@ export PATTERN=${PATTERN:-"$*"} + [ "$SLOW" = "no" ] && EXCEPT_SLOW="" + # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! + +-LUSTRE=${LUSTRE:-$(dirname $0)/..} ++LUSTRE=${LUSTRE:-$(dirname "$0")/..} + +-. $LUSTRE/tests/test-framework.sh +-init_test_env $@ +-. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} ++. "$LUSTRE/tests/test-framework.sh" ++init_test_env "$@" ++. "${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}" + init_logging + + # bug number for skipped test: +@@ -25,7 +32,7 @@ export LNETCTL=${LNETCTL:-"$LUSTRE/../lnet/utils/lnetctl"} + [[ -z $LNETCTL ]] && skip "Need lnetctl" + + restore_mounts=false +-if is_mounted $MOUNT || is_mounted $MOUNT2; then ++if is_mounted "$MOUNT" || is_mounted "$MOUNT2"; then + cleanupall || error "Failed cleanup prior to test execution" + restore_mounts=true + fi +@@ -62,19 +69,19 @@ set_env_vars_on_remote() { + + set_env_vars_on_remote + +-rm /tmp/tf.skip ++rm -f /tmp/tf.skip + + # do not exit if LUTF fails + set +e + + echo "+++++++++++STARTING LUTF" +-. $LUSTRE/tests/lutf/python/config/lutf_start.sh $CONFIG ++. "$LUSTRE/tests/lutf/python/config/lutf_start.sh" "$CONFIG" + rc=$? + echo "-----------STOPPING LUTF: $rc" + + if [ -d /tmp/lutf/ ]; then + tar -czf /tmp/lutf.tar.gz /tmp/lutf +- mv /tmp/lutf.tar.gz $LOGDIR ++ mv /tmp/lutf.tar.gz "$LOGDIR" + fi + + +diff --git a/lustre/tests/lutf/python/config/lutf_start.sh b/lustre/tests/lutf/python/config/lutf_start.sh +index 88a6c4adf8..391cb4076c 100644 +--- a/lustre/tests/lutf/python/config/lutf_start.sh ++++ b/lustre/tests/lutf/python/config/lutf_start.sh +@@ -1,3 +1,4 @@ ++#!/bin/bash + # note the cfg/.sh should export all environment variables + # required. EX: export ost4_HOST=lustre01 + +@@ -26,4 +27,4 @@ if [[ -z "${PYTHONBIN}" ]]; then + fi + + export LUTF_ENV_VARS=$1 +-$PYTHONBIN $LUSTRE/tests/lutf/python/config/lutf_start.py ++$PYTHONBIN "$LUSTRE/tests/lutf/python/config/lutf_start.py" +-- +2.33.0 + diff --git a/0026-LU-16382-config-ensure-lutf.sh-is-included-in-dist.patch b/0026-LU-16382-config-ensure-lutf.sh-is-included-in-dist.patch new file mode 100644 index 0000000000000000000000000000000000000000..51fa3fa769631e990514c221eb435e5622329fd5 --- /dev/null +++ b/0026-LU-16382-config-ensure-lutf.sh-is-included-in-dist.patch @@ -0,0 +1,56 @@ +From e24a797575bcf5c8a221e9171fefe6a6fc263121 Mon Sep 17 00:00:00 2001 +From: Mr NeilBrown +Date: Thu, 21 Mar 2024 12:39:11 -0700 +Subject: [PATCH 26/49] LU-16382 config: ensure lutf.sh is included in dist + +The official 2.15.1 source distribution does not contain +lutf.sh. As lustre.spec lists it (when LUTF is enabled) this causes a +build error. + +It is likely not included because "./configure --enable-dist" was run +in a context where swig was not installed. + +So when determining whether to enable lutf, first check for +enable_dist and in the case for enable_lutf="yes" + +Lustre-change: https://review.whamcloud.com/49382 +Lustre-commit: b486347abdb2f0c12f01deb5424c41a532c952a8 + +Test-Parameters: trivial +Signed-off-by: Mr NeilBrown +Change-Id: If5f856985a6d642822baba4b6ee301c04f851217 +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53299 +Reviewed-by: Jian Yu +Reviewed-by: Oleg Drokin +Reviewed-by: Timothy Day +Tested-by: jenkins +Tested-by: Maloo +--- + config/lustre-build.m4 | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/config/lustre-build.m4 b/config/lustre-build.m4 +index a81ce2749b..3d80397a34 100644 +--- a/config/lustre-build.m4 ++++ b/config/lustre-build.m4 +@@ -225,11 +225,15 @@ AC_ARG_ENABLE([tests], + # Check to see if we can build the lutf + # + AX_PYTHON_DEVEL() +-AS_IF([test "x$PYTHON_VERSION_CHECK" = xno], [ +- enable_lutf="no" ++AS_IF([test "x$enable_dist" != xno], [ ++ enable_lutf="yes" + ], [ ++ AS_IF([test "x$PYTHON_VERSION_CHECK" = xno], [ ++ enable_lutf="no" ++ ], [ + AX_PKG_SWIG(2.0, [ enable_lutf="yes" ], + [ enable_lutf="no" ]) ++ ]) + ]) + + AC_MSG_RESULT([$enable_tests]) +-- +2.33.0 + diff --git a/0026-LU-16934-kernel-update-RHEL-8.8-4.18.0-477.15.1.el8_.patch b/0026-LU-16934-kernel-update-RHEL-8.8-4.18.0-477.15.1.el8_.patch deleted file mode 100644 index 130c8b1c5cf4f9f710106b3ce9b070c404217213..0000000000000000000000000000000000000000 --- a/0026-LU-16934-kernel-update-RHEL-8.8-4.18.0-477.15.1.el8_.patch +++ /dev/null @@ -1,79 +0,0 @@ -From f144190bbe3728dc15fc731612f10fe3fad98bef Mon Sep 17 00:00:00 2001 -From: Jian Yu -Date: Fri, 28 Jul 2023 20:36:03 -0700 -Subject: [PATCH 26/61] LU-16934 kernel: update RHEL 8.8 - [4.18.0-477.15.1.el8_8] - -Update RHEL 8.8 kernel to 4.18.0-477.15.1.el8_8. - -Lustre-change: https://review.whamcloud.com/51517 -Lustre-commit: 830bf7a1f8de73a4f46248e6b8d2bbcd944a1f09 - -Test-Parameters: trivial fstype=ldiskfs \ -clientdistro=el8.8 serverdistro=el8.8 testlist=sanity - -Test-Parameters: trivial fstype=zfs \ -clientdistro=el8.8 serverdistro=el8.8 testlist=sanity - -Change-Id: I66365dce63065a0a07958a182a3c705e9948d424 -Signed-off-by: Jian Yu -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51518 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Yang Sheng -Reviewed-by: xinliang -Reviewed-by: Oleg Drokin ---- - lustre/ChangeLog | 4 ++-- - lustre/kernel_patches/targets/4.18-rhel8.8.target.in | 2 +- - lustre/kernel_patches/which_patch | 2 +- - 3 files changed, 4 insertions(+), 4 deletions(-) - -diff --git a/lustre/ChangeLog b/lustre/ChangeLog -index 57699d2232..4b21088869 100644 ---- a/lustre/ChangeLog -+++ b/lustre/ChangeLog -@@ -3,7 +3,7 @@ TBD Whamcloud - * See https://wiki.whamcloud.com/display/PUB/Lustre+Support+Matrix - for currently supported client and server kernel versions. - * Server primary kernels built and tested during release cycle: -- 4.18.0-477.10.1.el8 (RHEL8.8) -+ 4.18.0-477.15.1.el8 (RHEL8.8) - * Other server kernels known to build and work at some point (others may also work): - 3.10.0-862.14.4.el7 (RHEL7.5) - 3.10.0-957.27.2.el7 (RHEL7.6) -@@ -30,7 +30,7 @@ TBD Whamcloud - * Client primary kernels built and tested during release cycle: - 5.14.0-284.11.1.el9 (RHEL9.2) - 5.14.0-162.23.1.el9 (RHEL9.1) -- 4.18.0-477.10.1.el8 (RHEL8.8) -+ 4.18.0-477.15.1.el8 (RHEL8.8) - 5.4.0-37 (Ubuntu 20.04) - 5.14.21-150400.24.28 (SLES15 SP4) - * Other clients known to build on these kernels at some point (others may also work): -diff --git a/lustre/kernel_patches/targets/4.18-rhel8.8.target.in b/lustre/kernel_patches/targets/4.18-rhel8.8.target.in -index 172b7ca8c9..215a468fe4 100644 ---- a/lustre/kernel_patches/targets/4.18-rhel8.8.target.in -+++ b/lustre/kernel_patches/targets/4.18-rhel8.8.target.in -@@ -1,5 +1,5 @@ - lnxmaj="4.18.0" --lnxrel="477.10.1.el8_8" -+lnxrel="477.15.1.el8_8" - - KERNEL_SRPM=kernel-${lnxmaj}-${lnxrel}.src.rpm - SERIES=4.18-rhel8.8.series -diff --git a/lustre/kernel_patches/which_patch b/lustre/kernel_patches/which_patch -index 0047bf08eb..50ec106010 100644 ---- a/lustre/kernel_patches/which_patch -+++ b/lustre/kernel_patches/which_patch -@@ -25,6 +25,6 @@ PATCH SERIES FOR SERVER KERNELS: - 4.18-rhel8.5.series 4.18.0-348.23.1.el8 (RHEL 8.5) - 4.18-rhel8.6.series 4.18.0-372.32.1.el8 (RHEL 8.6) - 4.18-rhel8.7.series 4.18.0-425.3.1.el8 (RHEL 8.7) --4.18-rhel8.8.series 4.18.0-477.10.1.el8 (RHEL 8.8) -+4.18-rhel8.8.series 4.18.0-477.15.1.el8 (RHEL 8.8) - - See lustre/ChangeLog for supported client kernel versions. --- -2.33.0 - diff --git a/0027-LU-15515-contrib-epython-scripts-to-python3-conversi.patch b/0027-LU-15515-contrib-epython-scripts-to-python3-conversi.patch new file mode 100644 index 0000000000000000000000000000000000000000..f3dd764e40f72f4aa61727d1ccb9f782333ae79a --- /dev/null +++ b/0027-LU-15515-contrib-epython-scripts-to-python3-conversi.patch @@ -0,0 +1,1125 @@ +From 692ce6056c6081c4ff79f3116da862ce8446c2b5 Mon Sep 17 00:00:00 2001 +From: Oleg Drokin +Date: Thu, 21 Mar 2024 12:52:21 -0700 +Subject: [PATCH 27/49] LU-15515 contrib: epython scripts to python3 conversion + +This is to allow them to work with newer pykdump versions +that are actually current and therefore desirable. + +Lustre-change: https://review.whamcloud.com/46442 +Lustre-commit: 4f5eb6cfeae20dd14a1eb441515892ac8c27d196 + +Test-Parameters: trivial +Change-Id: I84a1be5f24c04b61d2ab1f8d1744e3ef752ce8c6 +Signed-off-by: Oleg Drokin +Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54522 +Tested-by: jenkins +Tested-by: Maloo +--- + .../debug_tools/epython_scripts/cfs_hashes.py | 30 ++-- + .../debug_tools/epython_scripts/cfs_hnodes.py | 8 +- + .../epython_scripts/crashlib/addrlib.py | 2 +- + .../crashlib/input/__init__.py | 16 +-- + .../crashlib/input/uflookup.py | 2 +- + .../epython_scripts/crashlib/page.py | 22 --- + .../epython_scripts/debug_flags.py | 6 +- + contrib/debug_tools/epython_scripts/dk.py | 23 ++-- + .../epython_scripts/jiffies2date.py | 2 +- + .../epython_scripts/ldlm_dumplocks.py | 129 ++++++++++-------- + .../epython_scripts/ldlm_lockflags.py | 4 +- + .../debug_tools/epython_scripts/lu_object.py | 52 +++---- + .../debug_tools/epython_scripts/lustrelib.py | 11 +- + .../debug_tools/epython_scripts/obd_devs.py | 25 ++-- + contrib/debug_tools/epython_scripts/ptlrpc.py | 34 ++--- + .../debug_tools/epython_scripts/rpc_opcode.py | 4 +- + .../debug_tools/epython_scripts/rpc_stats.py | 40 +++--- + .../debug_tools/epython_scripts/sbi_ptrs.py | 6 +- + .../epython_scripts/uniqueStacktrace.py | 10 +- + 19 files changed, 215 insertions(+), 211 deletions(-) + +diff --git a/contrib/debug_tools/epython_scripts/cfs_hashes.py b/contrib/debug_tools/epython_scripts/cfs_hashes.py +index 4ca16fb844..2445087730 100644 +--- a/contrib/debug_tools/epython_scripts/cfs_hashes.py ++++ b/contrib/debug_tools/epython_scripts/cfs_hashes.py +@@ -34,7 +34,7 @@ def cfs_hash_format_theta(theta): + + def print_theta(hs): + theta = cfs_hash_cur_theta(hs) +- print "Theta: %d %s" % (theta, cfs_hash_format_theta(theta)) ++ print("Theta: %d %s" % (theta, cfs_hash_format_theta(theta))) + + def print_thetas(name, hashtable): + hs = readSU('struct cfs_hash', hashtable) +@@ -45,20 +45,20 @@ def print_separator(count): + s = "" + for idx in xrange(count): + s += "=" +- print s ++ print(s) + + def print_hash_labels(): +- print "%-15s %-17s\t %-5s %-5s %-5s %-5s %-5s %-5s %-5s " \ ++ print("%-15s %-17s\t %-5s %-5s %-5s %-5s %-5s %-5s %-5s " \ + "%-5s %-5s %-5s %-5s %-11s %-11s %-11s %-5s" % \ + ("name", "cfs_hash", "cnt", "rhcnt", "xtr", "cur", "min", "max", "rhash", \ +- "bkt", "nbkt", "nhlst", "flags", "theta", "minT", "maxT", "bktsz") ++ "bkt", "nbkt", "nhlst", "flags", "theta", "minT", "maxT", "bktsz")) + + def print_hash_summary(name, hashtable): + hs = readSU('struct cfs_hash', hashtable) + if hs: + hs_cnt = readSU('atomic_t', hs.hs_count).counter + hs_ref = readSU('atomic_t', hs.hs_refcount).counter +- print "%-15s %-17x\t %-5d %-5d %-5d %-5d %-5d %-5d %-5d %-5d %-5d %-5d %-5x %-11s %-11s %-11s %-5d" % \ ++ print("%-15s %-17x\t %-5d %-5d %-5d %-5d %-5d %-5d %-5d %-5d %-5d %-5d %-5x %-11s %-11s %-11s %-5d" % \ + (name, (Addr(hs)), \ + readSU('atomic_t', hs.hs_count).counter, \ + hs.hs_rehash_count, \ +@@ -74,13 +74,13 @@ def print_hash_summary(name, hashtable): + cfs_hash_format_theta(cfs_hash_cur_theta(hs)), \ + cfs_hash_format_theta(hs.hs_min_theta), \ + cfs_hash_format_theta(hs.hs_max_theta), \ +- ll.cfs_hash_bucket_size(hs)) ++ ll.cfs_hash_bucket_size(hs))) + else: +- print "%-15s %-17x" % \ +- (name, (Addr(hs))) ++ print("%-15s %-17x" % \ ++ (name, (Addr(hs)))) + + def obd_print_export_hashes(obd, exp_list, fld): +- print "\nExport list head %x %s" % (exp_list, fld) ++ print("\nExport list head %x %s" % (exp_list, fld)) + for exp in readSUListFromHead(exp_list, fld, 'struct obd_export'): + print_hash_summary('exp_lock', exp.exp_lock_hash) + print_hash_summary('exp_flock', exp.exp_flock_hash) +@@ -88,10 +88,10 @@ def obd_print_export_hashes(obd, exp_list, fld): + def obd_print_one_device_hashes(obd): + try: + nm = ll.obd2str(obd) +- except Exception, e: ++ except Exception as e: + return 1 + +- print "obd_device %-17x %-22s" % (Addr(obd), ll.obd2str(obd)) ++ print("obd_device %-17x %-22s" % (Addr(obd), ll.obd2str(obd))) + print_hash_labels() + + print_hash_summary("uuid", obd.obd_uuid_hash) +@@ -108,7 +108,7 @@ def obd_print_one_device_hashes(obd): + + # obd_print_export_hashes(obd, obd.obd_exports, 'exp_obd_chain') + # obd_print_export_hashes(obd, obd.obd_exports_timed, 'exp_obd_chain_timed') +- print "" ++ print("") + return 0 + + def obd_devs_hash(): +@@ -121,7 +121,7 @@ def obd_devs_hash(): + + def ldlm_print_ns_hashes(ns, type): + ns_list = readSymbol(ns) +- print "\n%s namespaces-resources" % type ++ print("\n%s namespaces-resources" % type) + print_hash_labels() + for ns in readSUListFromHead(ns_list, 'ns_list_chain', 'struct ldlm_namespace'): + nm = ll.obd2str(ns.ns_obd)[0:20] +@@ -137,7 +137,7 @@ def lu_sites_hashes(): + print_hash_labels() + for site in readSUListFromHead(lu_sites, 'ls_linkage', 'struct lu_site'): + print_hash_summary("lu_site_vvp", site.ls_obj_hash) +- print "" ++ print("") + + + def global_hashes(): +@@ -147,7 +147,7 @@ def global_hashes(): + print_hash_summary("jobid_hash", readSymbol('jobid_hash')) + if symbol_exists('cl_env_hash'): + print_hash_summary("cl_env_hash", readSymbol('cl_env_hash')) +- print "" ++ print("") + + if __name__ == "__main__": + description = "Displays summary of hash tables in 'obd_devs'" +diff --git a/contrib/debug_tools/epython_scripts/cfs_hnodes.py b/contrib/debug_tools/epython_scripts/cfs_hnodes.py +index b47a81df90..b1bd74b315 100644 +--- a/contrib/debug_tools/epython_scripts/cfs_hnodes.py ++++ b/contrib/debug_tools/epython_scripts/cfs_hnodes.py +@@ -40,24 +40,24 @@ def get_hash_object(hs, hnode): + def dump_hnodes(hs, hlist, hnode, depth=0, ruler=RULER): + while(hnode != hlist & hnode): + s = get_hash_object(hs, hnode) +- print "%*.*shlist_node 0x%x %s" % (depth, depth, ruler, Addr(hnode), s) ++ print("%*.*shlist_node 0x%x %s" % (depth, depth, ruler, Addr(hnode), s)) + hnode = hnode.next + + def dump_hlist(hs, hlist, depth=0, ruler=RULER): + if hlist.first: + hnode = hlist.first +- print "%*.*shlist_head 0x%x" % (depth, depth, ruler, Addr(hlist)) ++ print("%*.*shlist_head 0x%x" % (depth, depth, ruler, Addr(hlist))) + dump_hnodes(hs, hlist, hnode, depth+DEPTH, ruler) + + def dump_hash_bucket(hs, bd_bkt, depth=0, ruler=RULER): +- print "%*.*scfs_hash_bucket 0x%x" % (depth, depth, ruler, Addr(bd_bkt)) ++ print("%*.*scfs_hash_bucket 0x%x" % (depth, depth, ruler, Addr(bd_bkt))) + for bd_offset in range(ll.CFS_HASH_BKT_NHLIST(hs)): + hlist = ll.cfs_hash_hhead(hs, bd_bkt, bd_offset) + if hlist: + dump_hlist(hs, hlist, depth+DEPTH, ruler) + + def dump_hash_table(hs): +- print "cfs_hash@0x%x" % Addr(hs) ++ print("cfs_hash@0x%x" % Addr(hs)) + + for bd_bkt in ll.cfs_hash_get_buckets(hs): + dump_hash_bucket(hs, bd_bkt, DEPTH, RULER) +diff --git a/contrib/debug_tools/epython_scripts/crashlib/addrlib.py b/contrib/debug_tools/epython_scripts/crashlib/addrlib.py +index d06a7697e3..0724ddd27b 100644 +--- a/contrib/debug_tools/epython_scripts/crashlib/addrlib.py ++++ b/contrib/debug_tools/epython_scripts/crashlib/addrlib.py +@@ -19,4 +19,4 @@ def phys2pfn(physaddr): + + def pfn2phys(pfn): + """Convert a page offset into a physical address.""" +- return pfn << crashlib.cid.mdtbl.pageshift ++ return int(pfn) << int(crashlib.cid.mdtbl.pageshift) +diff --git a/contrib/debug_tools/epython_scripts/crashlib/input/__init__.py b/contrib/debug_tools/epython_scripts/crashlib/input/__init__.py +index 3e4f0d7f06..d283a13ed8 100644 +--- a/contrib/debug_tools/epython_scripts/crashlib/input/__init__.py ++++ b/contrib/debug_tools/epython_scripts/crashlib/input/__init__.py +@@ -68,7 +68,7 @@ def toint(string, base=default_bases, suffixes=binary_suffixes): + # a suffix happens to contain valid numeric characters, we'll + # try the numeric interpretation before we try their multiplier + # meaning, e.g. 'g' is a valid numeric value in base 17). +- for i in xrange(2): ++ for i in range(2): + for b in bases: + try: + return int(string, b) * multiplier +@@ -104,7 +104,7 @@ def hex2int(string): + return toint(string, base=[16, 0]) + + +-def to_rangelist(args, default=xrange(0), base=[0,16], ++def to_rangelist(args, default=range(0), base=[0,16], + suffixes=binary_suffixes): + """Convert a bunch of range list strings into a list of ranges + +@@ -133,16 +133,16 @@ def to_rangelist(args, default=xrange(0), base=[0,16], + fields = range_str.split('-', 1) + start = toint(fields[0], base, suffixes=suffixes) + end = toint(fields[1], base, suffixes=suffixes) + 1 +- ranges.append(xrange(start, end)) ++ ranges.append(range(start, end)) + elif "#" in range_str: + fields = range_str.split('#', 1) + start = toint(fields[0], base, suffixes=suffixes) + end = start + toint(fields[1], base, suffixes=suffixes) +- ranges.append(xrange(start, end)) ++ ranges.append(range(start, end)) + else: + start = toint(range_str, base, suffixes=suffixes) + end = start + 1 +- ranges.append(xrange(start, end)) ++ ranges.append(range(start, end)) + + return ranges + +@@ -213,9 +213,9 @@ if __name__ == '__main__': + """Test both iter_rangelist and the underlying to_rangelist.""" + def test_good_single_ranges(self): + self.assertEqual(list(iter_rangestr([])), []) +- self.assertEqual(list(iter_rangestr(['1-2'])), list(xrange(1,3))) +- self.assertEqual(list(iter_rangestr(['1#2'])), list(xrange(1,3))) +- self.assertEqual(list(iter_rangestr(['1'])), list(xrange(1,2))) ++ self.assertEqual(list(iter_rangestr(['1-2'])), list(range(1,3))) ++ self.assertEqual(list(iter_rangestr(['1#2'])), list(range(1,3))) ++ self.assertEqual(list(iter_rangestr(['1'])), list(range(1,2))) + + def test_good_multiple_ranges(self): + test_rangestrs = [ +diff --git a/contrib/debug_tools/epython_scripts/crashlib/input/uflookup.py b/contrib/debug_tools/epython_scripts/crashlib/input/uflookup.py +index 8f9d4b6e3a..f89bc9e565 100644 +--- a/contrib/debug_tools/epython_scripts/crashlib/input/uflookup.py ++++ b/contrib/debug_tools/epython_scripts/crashlib/input/uflookup.py +@@ -95,7 +95,7 @@ if __name__ == '__main__': + {"casesensitive": False, "prefixok": True}, + {"casesensitive": True, "prefixok": False}, + {"casesensitive": True, "prefixok": True}] +- for i in xrange(len(expectedlist)): ++ for i in range(len(expectedlist)): + e = expectedlist[i] + if e is None: + self.assertRaises(KeyError, +diff --git a/contrib/debug_tools/epython_scripts/crashlib/page.py b/contrib/debug_tools/epython_scripts/crashlib/page.py +index ef87d057d3..32acf0931b 100644 +--- a/contrib/debug_tools/epython_scripts/crashlib/page.py ++++ b/contrib/debug_tools/epython_scripts/crashlib/page.py +@@ -9,7 +9,6 @@ import crashlib.cid + import crashlib.cid.machdep_table + import crashlib.cid.page_flags + import crashlib.cid.phys_mem_map +-import crashlib.memarray + + # -------------------------------------------------------------------------- + +@@ -104,24 +103,3 @@ def pfn(page): + return (page - vmemmap_vaddr) / page_struct_size + + # -------------------------------------------------------------------------- +- +-def page_list(): +- """Return a list-like class of page structs indexed by pfn. +- +- This implementation assumes the kernel is configured with a virtually +- contiguous mem_map. +- """ +- # If the kernel doesn't have a virtually contiguous mem_map, this could +- # be changed to return a chained list of MemCArray objects. +- +- PAGE_SHIFT = crashlib.cid.mdtbl.pageshift +- pfn_start = crashlib.cid.physmap[0].start >> PAGE_SHIFT +- pfn_end = crashlib.cid.physmap[-1].end >> PAGE_SHIFT +- +- # Find page map and create an array of page_struct +- vmemmap_addr = crashlib.cid.mdtbl.vmemmap_vaddr +- +- return crashlib.memarray.MemCArray(vmemmap_addr, +- lambda a:readSU('struct page',a), +- getSizeOf('struct page'), +- pfn_end-pfn_start) +diff --git a/contrib/debug_tools/epython_scripts/debug_flags.py b/contrib/debug_tools/epython_scripts/debug_flags.py +index ddfb376497..1ecf175065 100644 +--- a/contrib/debug_tools/epython_scripts/debug_flags.py ++++ b/contrib/debug_tools/epython_scripts/debug_flags.py +@@ -53,12 +53,12 @@ def print_flags(flag_tbl, mask): + if key & mask: + flags = flags + value + " " + tmp &= ~key +- print "mask: 0x%x = %s" % (mask, flags) ++ print("mask: 0x%x = %s" % (mask, flags)) + if tmp != 0: +- print "unknown bits set in mask: 0x%x" % tmp ++ print("unknown bits set in mask: 0x%x" % tmp) + + def dump_debug_flags(bitmask): +- print bitmask ++ print(bitmask) + if not bitmask: + bitmask = readSymbol('libcfs_debug') + print_flags(debug_flags_tbl, bitmask) +diff --git a/contrib/debug_tools/epython_scripts/dk.py b/contrib/debug_tools/epython_scripts/dk.py +index 1806e9a726..de1ae5bce8 100644 +--- a/contrib/debug_tools/epython_scripts/dk.py ++++ b/contrib/debug_tools/epython_scripts/dk.py +@@ -17,7 +17,8 @@ import os + description_short = 'Dump and sort the Lustre dk logs.' + + def do_shell_cmd(cmd): +- return os.popen(cmd).read() ++ return "done" ++ #return os.popen(cmd).read() + + # --------------------------------------------------------------------------- + # pfn: 2582e8c, physaddr: 2582e8c000, vaddr: ffff002582e8c000 +@@ -32,12 +33,12 @@ def dump_dk_line(tmpfd, options, pfn, used): + laddr = vaddr + hdr_size + try: + line = readmem(laddr, hdr.ph_len - hdr_size) +- except: +- print "Skipping pfn: %x, physaddr: %x, vaddr: %x, laddr: %x" % \ +- (pfn, physaddr, vaddr, laddr) ++ except: ++ print("Skipping pfn: %x, physaddr: %x, vaddr: %x, laddr: %x" % \ ++ (pfn, physaddr, vaddr, laddr)) + return + +- (filename,function,text) = line.split('\0') ++ (filename,function,text) = line.decode().split('\0') + text = text.rstrip() + + used -= hdr.ph_len +@@ -68,8 +69,8 @@ def walk_pages(tmpfd, options, cfs_page_head, trace_page_struct): + def walk_array(options): + """Walk the cfs_trace_data array of array pointers.""" + +- fname = do_shell_cmd('mktemp .dklogXXXX').rstrip() +- tmpfd = file(fname, 'w') ++ #fname = do_shell_cmd('mktemp .dklogXXXX').rstrip() ++ tmpfd = open("/tmp/log.bin", 'w') + + try: + cfs_trace_data = readSymbol('cfs_trace_data') +@@ -79,20 +80,20 @@ def walk_array(options): + cfs_trace_data = readSymbol('trace_data') + trace_page_struct = 'struct trace_page' + except: +- print "Ensure you have loaded the Lustre modules" ++ print("Ensure you have loaded the Lustre modules") + return 1 + + for cfstd_array in cfs_trace_data: + if not cfstd_array: continue + +- for i in xrange(sys_info.CPUS): ++ for i in range(sys_info.CPUS): + u = cfstd_array[i] + walk_pages(tmpfd, options, u.tcd.tcd_pages, trace_page_struct) + walk_pages(tmpfd, options, u.tcd.tcd_stock_pages, trace_page_struct) + + tmpfd.close() +- print do_shell_cmd('sort -n -s -t: -k4,4 ' + fname) +- print do_shell_cmd('rm ' + fname) ++ print(do_shell_cmd('sort -n -s -t: -k4,4 ' + fname)) ++ print(do_shell_cmd('rm ' + fname)) + + # --------------------------------------------------------------------------- + def dump_dk_log(): +diff --git a/contrib/debug_tools/epython_scripts/jiffies2date.py b/contrib/debug_tools/epython_scripts/jiffies2date.py +index 152c00f35e..2604ee2907 100644 +--- a/contrib/debug_tools/epython_scripts/jiffies2date.py ++++ b/contrib/debug_tools/epython_scripts/jiffies2date.py +@@ -25,7 +25,7 @@ def jiffies2date(jts): + + stime = scur - soffset + date = time.asctime(time.localtime(stime)) +- print '%s (epoch: %d)' % (date, stime) ++ print('%s (epoch: %d)' % (date, stime)) + + if __name__ == "__main__": + description = "Print the date and time of a given jiffies timestamp. " + \ +diff --git a/contrib/debug_tools/epython_scripts/ldlm_dumplocks.py b/contrib/debug_tools/epython_scripts/ldlm_dumplocks.py +index 2f695d2239..5df8866fca 100644 +--- a/contrib/debug_tools/epython_scripts/ldlm_dumplocks.py ++++ b/contrib/debug_tools/epython_scripts/ldlm_dumplocks.py +@@ -40,85 +40,104 @@ def ldlm_dump_lock(lock, pos, lstname): + obd = None + imp = None + if(lock == None): +- print " NULL LDLM lock" ++ print(" NULL LDLM lock") ++ return ++ try: ++ if lock.l_handle.hasField("h_ref"): ++ refcounter = lock.l_handle.h_ref.refs ++ else: ++ refcounter = lock.l_refc.counter ++ ++ print(" -- Lock: (ldlm_lock) %#x/%#x (rc: %d) (pos: %d/%s) (pid: %d)" % \ ++ (Addr(lock), lock.l_handle.h_cookie, refcounter, ++ pos, lstname, lock.l_pid)) ++ if(lock.l_conn_export): ++ obd = lock.l_conn_export.exp_obd ++ if(lock.l_export and lock.l_export.exp_connection): ++ print(" Node: NID %s (remote: %#x) export" % \ ++ (ll.nid2str(lock.l_export.exp_connection.c_peer.nid), ++ lock.l_remote_handle.cookie)) ++ elif(obd == None): ++ print(" Node: local") ++ else: ++ imp = obd.u.cli.cl_import ++ print(" Node: NID %s (remote: %#x) import " % \ ++ (ll.nid2str(imp.imp_connection.c_peer.nid), ++ lock.l_remote_handle.cookie)) ++ ++ res = lock.l_resource ++ print(" Resource: %#x [0x%x:0x%x:0x%x].%x" % \ ++ (Addr(res), ++ res.lr_name.name[0], ++ res.lr_name.name[1], ++ res.lr_name.name[2], ++ res.lr_name.name[3])) ++ ++ print(" Req mode: %s, grant mode: %s, rc: %d, read: %d, \ ++ write: %d flags: %#x" % (lockmode2str(lock.l_req_mode), ++ lockmode2str(lock.l_granted_mode), ++ refcounter, lock.l_readers, lock.l_writers, ++ lock.l_flags)) ++ ++ lr_type = lock.l_resource.lr_type ++ if(lr_type == enum_LDLM_EXTENT): ++ print(" Extent: %d -> %d (req %d-%d)" % \ ++ (lock.l_policy_data.l_extent.start, ++ lock.l_policy_data.l_extent.end, ++ lock.l_req_extent.start, lock.l_req_extent.end)) ++ elif(lr_type == enum_LDLM_FLOCK): ++ print(" Pid: %d Flock: 0x%x -> 0x%x" % \ ++ (lock.l_policy_data.l_flock.pid, ++ lock.l_policy_data.l_flock.start, ++ lock.l_policy_data.l_flock.end)) ++ elif(lr_type == enum_LDLM_IBITS): ++ print(" Bits: %#x" % \ ++ (lock.l_policy_data.l_inodebits.bits)) ++ except (crash.error, IndexError): ++ print(" Corrupted lock %#x" % (Addr(lock))) + return +- print " -- Lock: (ldlm_lock) %#x/%#x (rc: %d) (pos: %d/%s) (pid: %d)" % \ +- (Addr(lock), lock.l_handle.h_cookie, lock.l_refc.counter, +- pos, lstname, lock.l_pid) +- if(lock.l_conn_export): +- obd = lock.l_conn_export.exp_obd +- if(lock.l_export and lock.l_export.exp_connection): +- print " Node: NID %s (remote: %#x) export" % \ +- (ll.nid2str(lock.l_export.exp_connection.c_peer.nid), +- lock.l_remote_handle.cookie) +- elif(obd == None): +- print " Node: local" +- else: +- imp = obd.u.cli.cl_import +- print " Node: NID %s (remote: %#x) import " % \ +- (ll.nid2str(imp.imp_connection.c_peer.nid), +- lock.l_remote_handle.cookie) +- +- res = lock.l_resource +- print " Resource: %#x [0x%x:0x%x:0x%x].%x" % \ +- (Addr(res), +- res.lr_name.name[0], +- res.lr_name.name[1], +- res.lr_name.name[2], +- res.lr_name.name[3]) +- +- print " Req mode: %s, grant mode: %s, rc: %d, read: %d, \ +- write: %d flags: %#x" % (lockmode2str(lock.l_req_mode), +- lockmode2str(lock.l_granted_mode), +- lock.l_refc.counter, lock.l_readers, lock.l_writers, +- lock.l_flags) +- +- lr_type = lock.l_resource.lr_type +- if(lr_type == enum_LDLM_EXTENT): +- print " Extent: %d -> %d (req %d-%d)" % \ +- (lock.l_policy_data.l_extent.start, +- lock.l_policy_data.l_extent.end, +- lock.l_req_extent.start, lock.l_req_extent.end) +- elif(lr_type == enum_LDLM_FLOCK): +- print " Pid: %d Flock: 0x%x -> 0x%x" % \ +- (lock.l_policy_data.l_flock.pid, +- lock.l_policy_data.l_flock.start, +- lock.l_policy_data.l_flock.end) +- elif(lr_type == enum_LDLM_IBITS): +- print " Bits: %#x" % \ +- (lock.l_policy_data.l_inodebits.bits) + + def ldlm_dump_resource(res): + res_lr_granted = readSU('struct list_head', Addr(res.lr_granted)) + res_lr_waiting = readSU('struct list_head', Addr(res.lr_waiting)) +- print "-- Resource: (ldlm_resource) %#x [0x%x:0x%x:0x%x].%x (rc: %d)" % \ +- (Addr(res), res.lr_name.name[0], res.lr_name.name[1], +- res.lr_name.name[2], res.lr_name.name[3], res.lr_refcount.counter) ++ try: ++ print("-- Resource: (ldlm_resource) %#x [0x%x:0x%x:0x%x].%x (rc: %d)" % \ ++ (Addr(res), res.lr_name.name[0], res.lr_name.name[1], ++ res.lr_name.name[2], res.lr_name.name[3], res.lr_refcount.counter)) ++ except (crash.error, IndexError): ++ print("-- Corrupted resource %#x" % (Addr(res))) ++ return + if not ll.list_empty(res_lr_granted): + pos = 0 +- print " Granted locks: " ++ print(" Granted locks: ") + tmp = res_lr_granted.next + while(tmp != res_lr_granted): + pos += 1 + lock = readSU('struct ldlm_lock', + Addr(tmp)-member_offset('struct ldlm_lock', 'l_res_link')) + ldlm_dump_lock(lock, pos, "grnt") +- tmp = tmp.next ++ try: ++ tmp = tmp.next ++ except (crash.error, IndexError): ++ break + if not ll.list_empty(res_lr_waiting): + pos = 0 +- print " Waiting locks: " ++ print(" Waiting locks: ") + tmp = res_lr_waiting.next + while(tmp != res_lr_waiting): + pos += 1 + lock = readSU('struct ldlm_lock', + Addr(tmp)-member_offset('struct ldlm_lock', 'l_res_link')) + ldlm_dump_lock(lock, pos, "wait") +- tmp = tmp.next ++ try: ++ tmp = tmp.next ++ except (crash.error, IndexError): ++ break + + def print_namespace(ns, client_server): +- print "Namespace: (ldlm_namespace) %#x, %s\t(rc: %d, side: %s)\tpoolcnt: %d unused: %d" % \ ++ print("Namespace: (ldlm_namespace) %#x, %s\t(rc: %d, side: %s)\tpoolcnt: %d unused: %d" % \ + (Addr(ns), ll.obd2str(ns.ns_obd), ns.ns_bref.counter, +- client_server, ns.ns_pool.pl_granted.counter, ns.ns_nr_unused) ++ client_server, ns.ns_pool.pl_granted.counter, ns.ns_nr_unused)) + + def ldlm_dump_ns_resources(ns): + if args.nflag: +diff --git a/contrib/debug_tools/epython_scripts/ldlm_lockflags.py b/contrib/debug_tools/epython_scripts/ldlm_lockflags.py +index 5788e3ee85..d2b68fb653 100644 +--- a/contrib/debug_tools/epython_scripts/ldlm_lockflags.py ++++ b/contrib/debug_tools/epython_scripts/ldlm_lockflags.py +@@ -70,9 +70,9 @@ def print_flags(flag_dict, mask): + if key & mask: + flags = flags + value + " " + tmp &= ~key +- print "mask: 0x%x = %s" % (mask, flags) ++ print("mask: 0x%x = %s" % (mask, flags)) + if tmp != 0: +- print "unknown bits set in mask: 0x%x" % tmp ++ print("unknown bits set in mask: 0x%x" % tmp) + + if __name__ == "__main__": + description = "Prints string identifiers for specified LDLM flags." +diff --git a/contrib/debug_tools/epython_scripts/lu_object.py b/contrib/debug_tools/epython_scripts/lu_object.py +index 12a669b437..6110d7e07a 100644 +--- a/contrib/debug_tools/epython_scripts/lu_object.py ++++ b/contrib/debug_tools/epython_scripts/lu_object.py +@@ -31,7 +31,7 @@ IDIF_OID_MAX_BITS = 48 + IDIF_OID_MASK = ((1 << IDIF_OID_MAX_BITS) -1) + + def lov_print_empty(obj, depth=0, ruler=RULER): +- print "empty %d" % obj.lo_layout_invalid ++ print("empty %d" % obj.lo_layout_invalid) + + def lov_print_raid0(obj, depth=0, ruler=RULER): + r0 = None +@@ -41,7 +41,7 @@ def lov_print_raid0(obj, depth=0, ruler=RULER): + stripes = lsm.lsm_stripe_count + layout_gen = lsm.lsm_layout_gen + pattern = lsm.lsm_pattern +- except Exception, e: ++ except Exception as e: + magic = lsm.lsm_wire.lw_magic + stripes = lsm.lsm_wire.lw_stripe_count + layout_gen = lsm.lsm_wire.lw_layout_gen +@@ -51,17 +51,17 @@ def lov_print_raid0(obj, depth=0, ruler=RULER): + lli = readU32(Addr(obj) + member_offset('struct lov_object', 'lo_layout_invalid')) + invalid = "invalid" if lli else "valid" + if r0 and r0.lo_nr: +- print "%*.*sstripes: %d, %s, lsm[0x%x 0x%X %d %d %d %d]:" % \ ++ print("%*.*sstripes: %d, %s, lsm[0x%x 0x%X %d %d %d %d]:" % \ + (depth, depth, ruler, + r0.lo_nr, invalid, Addr(lsm), magic, +- lsm.lsm_refc.counter, stripes, layout_gen, pattern) ++ lsm.lsm_refc.counter, stripes, layout_gen, pattern)) + for i in range(r0.lo_nr): + los = r0.lo_sub[i] + if los: + sub = los.lso_cl.co_lu + lovsub_object_print(sub, depth+DEPTH, ruler) + else: +- print "sub %d absent" % i ++ print("sub %d absent" % i) + + def lov_print_released(obj, depth=0, ruler=RULER): + lsm = obj.lo_lsm +@@ -71,10 +71,10 @@ def lov_print_released(obj, depth=0, ruler=RULER): + lli = readU32(Addr(obj) + member_offset('struct lov_object', 'lo_layout_invalid')) + invalid = "invalid" if lli else "valid" + if magic==LOV_MAGIC or magic==LOV_MAGIC_V3: +- print "%*.*sreleased: %s, lov_stripe_md: 0x%x [0x%X %d %u %u]:" % \ ++ print("%*.*sreleased: %s, lov_stripe_md: 0x%x [0x%X %d %u %u]:" % \ + (depth, depth, ruler, + invalid, Addr(lsm), magic, lsm.lsm_refc.counter, +- entries, layout_gen) ++ entries, layout_gen)) + + LOV_PRINT_TYPE = { + 0:lov_print_empty, +@@ -83,15 +83,15 @@ LOV_PRINT_TYPE = { + + def vvp_object_print(o, depth=0, ruler=RULER): + obj = readSU('struct vvp_object', Addr(o) - member_offset('struct vvp_object', 'vob_cl.co_lu')) +- print "%*.*s(trans:%s mmap:%d) inode: 0x%x " % \ ++ print("%*.*s(trans:%s mmap:%d) inode: 0x%x " % \ + (depth, depth, ruler, + obj.vob_transient_pages.counter, + obj.vob_mmap_cnt.counter, +- Addr(obj.vob_inode)) ++ Addr(obj.vob_inode))) + + def lod_object_print(o, depth=0, ruler=RULER): + obj = readSU('struct lod_object', Addr(o) - member_offset('struct lod_object', 'ldo_obj.do_lu')) +- print "%*.*slod_object@0x%x" % (depth, depth, ruler, Addr(obj)) ++ print("%*.*slod_object@0x%x" % (depth, depth, ruler, Addr(obj))) + + def lov_object_print(o, depth=0, ruler=RULER): + obj = readSU('struct lov_object', Addr(o) - member_offset('struct lov_object', 'lo_cl.co_lu')) +@@ -100,31 +100,31 @@ def lov_object_print(o, depth=0, ruler=RULER): + + def lovsub_object_print(o, depth=0, ruler=RULER): + obj = readSU('struct lovsub_object', Addr(o) - member_offset('struct lovsub_object', 'lso_cl.co_lu')) +- print "%*.*slso_index: %d" % (depth, depth, ruler, obj.lso_index) ++ print("%*.*slso_index: %d" % (depth, depth, ruler, obj.lso_index)) + + def mdd_object_print(o, depth=0, ruler=RULER): + obj = readSU('struct mdd_object', Addr(o) - member_offset('struct mdd_object', 'mod_obj.mo_lu')) +- print "%*.*smdd_object@0x%x(open_count=%d, valid=%x, cltime=%u, flags=%x)" % \ ++ print("%*.*smdd_object@0x%x(open_count=%d, valid=%x, cltime=%u, flags=%x)" % \ + (depth, depth, ruler, Addr(obj), obj.mod_count, obj.mod_valid, +- obj.mod_cltime, obj.mod_flags) ++ obj.mod_cltime, obj.mod_flags)) + + def mdt_object_print(o, depth=0, ruler=RULER): + obj = readSU('struct mdt_object', Addr(o) - member_offset('struct mdt_object', 'mot_obj')) +- print "%*.*smdt_object@0x%x(ioepoch=%u, flags=%x, epochcount=%d, writecount-%d" % \ ++ print("%*.*smdt_object@0x%x(ioepoch=%u, flags=%x, epochcount=%d, writecount-%d" % \ + (depth, depth, ruler, Addr(obj), obj.mot_ioepoch, obj.mot_flags, +- obj.mot_ioepoch_count, obj.mot_writecount) ++ obj.mot_ioepoch_count, obj.mot_writecount)) + + def mgs_object_print(o, depth=0, ruler=RULER): + obj = readSU('struct mgs_object', Addr(o) - member_offset('struct mgs_object', 'mgo_obj.do_lu')) +- print "%*.*smgs_object@0x%x" % (depth, depth, ruler, Addr(obj)) ++ print("%*.*smgs_object@0x%x" % (depth, depth, ruler, Addr(obj))) + + def echo_object_print(o, depth=0, ruler=RULER): + clo = readSU('struct cl_object', Addr(o) - member_offset('struct cl_object', 'co_lu')) + obj = readSU('struct echo_object', Addr(clo) - member_offset('struct echo_object', 'eo_cl')) +- print "%*.*sechocl_object@0x%x" % (depth, depth, ruler, Addr(obj)) ++ print("%*.*sechocl_object@0x%x" % (depth, depth, ruler, Addr(obj))) + + def ofd_object_print(o, depth=0, ruler=RULER): +- print "%*.*sofd_object@0x%x" % (depth, depth, ruler, Addr(o)) ++ print("%*.*sofd_object@0x%x" % (depth, depth, ruler, Addr(o))) + + def osc_object_print(o, depth=0, ruler=RULER): + obj = readSU('struct osc_object', Addr(o) - member_offset('struct osc_object', 'oo_cl.co_lu')) +@@ -146,18 +146,18 @@ def osc_object_print(o, depth=0, ruler=RULER): + else: + ostid_seq = ostid.oi_fid.f_seq + ostid_id = ostid.oi_fid.f_oid +- print "%*.*sid: 0x%x:%u idx: %d gen: %d kms_valid: %u kms: %u rc: %d force_sync: %d min_xid: %u" % \ ++ print("%*.*sid: 0x%x:%u idx: %d gen: %d kms_valid: %u kms: %u rc: %d force_sync: %d min_xid: %u" % \ + (depth, depth, ruler, ostid_seq, ostid_id, + oinfo.loi_ost_idx, oinfo.loi_ost_gen, oinfo.loi_kms_valid, +- oinfo.loi_kms, ar.ar_rc, ar.ar_force_sync, ar.ar_min_xid) ++ oinfo.loi_kms, ar.ar_rc, ar.ar_force_sync, ar.ar_min_xid)) + + def osd_object_print(o, depth=0, ruler=RULER): + obj = readSU('struct osd_object', Addr(o) - member_offset('struct osd_object', 'oo_dt.do_lu')) +- print "%*.*sosd_object@0x%x" % (depth, depth, ruler, Addr(obj)) ++ print("%*.*sosd_object@0x%x" % (depth, depth, ruler, Addr(obj))) + + def osp_object_print(o, depth=0, ruler=RULER): + obj = readSU('struct osp_object', Addr(o) - member_offset('struct osp_object', 'opo_obj.do_lu')) +- print "%*.*sosp_object@0x%x" % (depth, depth, ruler, Addr(o)) ++ print("%*.*sosp_object@0x%x" % (depth, depth, ruler, Addr(o))) + + OBJ_PRINT = { + "vvp":vvp_object_print, +@@ -178,7 +178,7 @@ def print_object_from_name(name, obj, depth=0, ruler=RULER): + OBJ_PRINT[name](obj, depth, ruler) + + def print_object(pos, depth=0, ruler=RULER): +- print "%*.*s%s@0x%x" % (depth, depth, ruler, pos.lo_dev.ld_type.ldt_name, Addr(pos)) ++ print("%*.*s%s@0x%x" % (depth, depth, ruler, pos.lo_dev.ld_type.ldt_name, Addr(pos))) + if (pos.lo_ops.loo_object_print): + print_object_from_name(pos.lo_dev.ld_type.ldt_name, pos, depth+DEPTH, ruler) + +@@ -186,7 +186,7 @@ def print_object_from_header(loh, depth=0, ruler=RULER): + head = loh.loh_layers + empty = "" if (loh.loh_lru.next == loh.loh_lru) else " lru" + exists = " exist" if loh.loh_attr & LOHA_EXISTS else "" +- print "%*.*slu_object_header@0x%x[fl:0x%x, rc:%d, [0x%x:0x%x:0x%x]%s%s] {" % \ ++ print("%*.*slu_object_header@0x%x[fl:0x%x, rc:%d, [0x%x:0x%x:0x%x]%s%s] {" % \ + (depth, depth, ruler, + Addr(loh), + loh.loh_flags, +@@ -195,10 +195,10 @@ def print_object_from_header(loh, depth=0, ruler=RULER): + loh.loh_fid.f_oid, + loh.loh_fid.f_ver, + empty, +- exists) ++ exists)) + for obj in readSUListFromHead(head, 'lo_linkage', 'struct lu_object'): + print_object(obj, depth+DEPTH, ruler) +- print "%*.*s} header@0x%x\n" % (depth, depth, ruler, Addr(loh)) ++ print("%*.*s} header@0x%x\n" % (depth, depth, ruler, Addr(loh))) + + if __name__ == "__main__": + description = "Prints contents of an lu_object" +diff --git a/contrib/debug_tools/epython_scripts/lustrelib.py b/contrib/debug_tools/epython_scripts/lustrelib.py +index 879282ffc6..50b0f3ee47 100644 +--- a/contrib/debug_tools/epython_scripts/lustrelib.py ++++ b/contrib/debug_tools/epython_scripts/lustrelib.py +@@ -81,7 +81,10 @@ def cfs_hash_get_hlist_nodes(hsh, bd_bkt, bd_offset): + hnode = hlist.first + while (hnode and hnode != hlist): + hnodes.append(hnode) +- hnode = hnode.next ++ try: ++ hnode = hnode.next ++ except (crash.error, IndexError): ++ break + return hnodes + + def cfs_hash_get_nodes(hsh): +@@ -109,6 +112,7 @@ def LNET_NETNUM(net): + LNET_NID_ANY = 0xffffffffffffffff + LNET_NIDSTR_SIZE = 32 + ++SOCKLND = 2 + O2IBLND = 5 + PTLLND = 4 + GNILND = 13 +@@ -131,6 +135,9 @@ def nid2str(nid): + s = "%d@ptl" % addr + elif lnd == GNILND: + s = "%d@gni" % addr ++ elif lnd == SOCKLND: ++ s = "%d.%d.%d.%d@tcp" % ((addr >> 24) & 0xff, (addr >> 16) & 0xff, ++ (addr >> 8) & 0xff, addr & 0xff) + else: + nnum = 0 + if nnum != 0: +@@ -228,7 +235,7 @@ tmpsiz = 256 + LNET_CPT_BITS = the_lnet.ln_cpt_bits + LNET_PROC_CPT_BITS = LNET_CPT_BITS + 1 + LNET_LOFFT_BITS = getSizeOf('loff_t') * 8 +-LNET_PROC_VER_BITS = max((min(LNET_LOFFT_BITS, 64) / 4), 8) ++LNET_PROC_VER_BITS = int(max((min(LNET_LOFFT_BITS, 64) / 4), 8)) + LNET_PROC_HASH_BITS = 9 + LNET_PROC_HOFF_BITS = LNET_LOFFT_BITS - LNET_PROC_CPT_BITS - LNET_PROC_VER_BITS - LNET_PROC_HASH_BITS -1 + LNET_PROC_HPOS_BITS = LNET_PROC_HASH_BITS + LNET_PROC_HOFF_BITS +diff --git a/contrib/debug_tools/epython_scripts/obd_devs.py b/contrib/debug_tools/epython_scripts/obd_devs.py +index 6d8848151b..a6610df897 100644 +--- a/contrib/debug_tools/epython_scripts/obd_devs.py ++++ b/contrib/debug_tools/epython_scripts/obd_devs.py +@@ -17,12 +17,12 @@ description_short = "Displays the contents of global 'obd_devs'" + SEP_SIZE = 152 + def print_separator(count): + s="" +- for idx in xrange(count): ++ for idx in range(count): + s += "=" +- print s ++ print(s) + + def print_header(): +- print "%-19s %-22s \t%-22s %-19s %-19s %-12s %-10s %-7s %-10s" % \ ++ print("%-19s %-22s \t%-22s %-19s %-19s %-12s %-10s %-7s %-10s" % \ + ("obd_device", + "obd_name", + "ip_address", +@@ -31,7 +31,7 @@ def print_header(): + "imp_state", + "ish_time", + "index", +- "conn_cnt") ++ "conn_cnt")) + print_separator(SEP_SIZE) + + IMP_STATE = { +@@ -52,10 +52,10 @@ IMP_STATE = { + def print_one_device(obd, stats_flag): + try: + nid = ll.obd2nidstr(obd) +- except Exception, e: ++ except Exception as e: + try: +- print "0x%-17x %-22s" % (Addr(obd), ll.obd2str(obd)) +- except Exception, e: ++ print("0x%-17x %-22s" % (Addr(obd), ll.obd2str(obd))) ++ except Exception as e: + return 1 + return 0 + +@@ -68,11 +68,11 @@ def print_one_device(obd, stats_flag): + impstate=IMP_STATE.get(obd.u.cli.cl_import.imp_state) + index=obd.u.cli.cl_import.imp_state_hist_idx - 1 + if index > 0 and index < 16: +- ish_time=obd.u.cli.cl_import.imp_state_hist[index].ish_time +- inflight=obd.u.cli.cl_import.imp_inflight.counter ++ ish_time=obd.u.cli.cl_import.imp_state_hist[index].ish_time ++ inflight=obd.u.cli.cl_import.imp_inflight.counter + connect_cnt = obd.u.cli.cl_import.imp_conn_cnt + +- print "0x%-17x %-22s\t%-22s\t 0x%-17x 0x%-17x %-10s %-10d %5d %5d" % \ ++ print("0x%-17x %-22s\t%-22s\t 0x%-17x 0x%-17x %-10s %-10d %5d %5d" % \ + (Addr(obd), + ll.obd2str(obd), + nid, +@@ -81,10 +81,9 @@ def print_one_device(obd, stats_flag): + impstate, + ish_time, + index, +- connect_cnt) ++ connect_cnt)) + if stats_flag: +- print +- rs.osc_rpc_stats_seq_show(Addr(obd.u.cli)) ++ print(rs.osc_rpc_stats_seq_show(Addr(obd.u.cli))) + print_separator(SEP_SIZE) + return 0 + +diff --git a/contrib/debug_tools/epython_scripts/ptlrpc.py b/contrib/debug_tools/epython_scripts/ptlrpc.py +index 518ef3e877..bd7726d801 100644 +--- a/contrib/debug_tools/epython_scripts/ptlrpc.py ++++ b/contrib/debug_tools/epython_scripts/ptlrpc.py +@@ -19,16 +19,16 @@ description_short = "Displays the RPC queues of the Lustre ptlrpcd daemons" + + def print_separator(count): + s = "" +- for idx in xrange(count): ++ for idx in range(count): + s += "=" +- print s ++ print(s) + + def print_title(title): + if title: +- print "\n" + title +- print "%-14s %-6s %-19s %-18s %-19s %-4s %-14s %-4s %-22s %-19s" \ ++ print("\n" + title) ++ print("%-14s %-6s %-19s %-18s %-19s %-4s %-14s %-4s %-22s %-19s" \ + % ("thread", "pid", "ptlrpc_request", "xid", "nid", "opc", +- "phase:flags", "R:W", "sent/deadline", "ptlrpc_body") ++ "phase:flags", "R:W", "sent/deadline", "ptlrpc_body")) + print_separator(148) + + def enum(**enums): +@@ -81,7 +81,7 @@ def get_ptlrpc_body(req): + offset = size_round(offset) + addr = Addr(msg) + offset + if addr == 0: +- print "addr" ++ print("addr") + return None + return readSU('struct ptlrpc_body_v2', addr) + +@@ -154,7 +154,7 @@ def print_one_request(sthread, req): + nid = ll.nid2str(req.rq_import.imp_connection.c_peer.nid) + brw = "%1d:%1d" % (req.rq_bulk_read, req.rq_bulk_write) + rq_sent_dl = "%d/%d" % (req.rq_sent, req.rq_deadline) +- print "%-14s %-6s 0x%-17x %-18d %-19s %-4d %-14s %-4s %-22s 0x%-17x" % \ ++ print("%-14s %-6s 0x%-17x %-18d %-19s %-4d %-14s %-4s %-22s 0x%-17x" % \ + (sthread, + status, + Addr(req), +@@ -164,7 +164,7 @@ def print_one_request(sthread, req): + get_phase_flags(req), + brw, + rq_sent_dl, +- pbaddr) ++ pbaddr)) + + def print_request_list(sthread, lhdr, loffset): + try: +@@ -174,7 +174,7 @@ def print_request_list(sthread, lhdr, loffset): + req = readSU('struct ptlrpc_request', reqlnk.next-loffset) + print_one_request(sthread, req) + +- except Exception, e: ++ except Exception as e: + print_exc() + return 1 + return 0 +@@ -194,9 +194,9 @@ def foreach_ptlrpcd_ctl(callback, *args): + pinfo_rpcds = readSymbol('ptlrpcds') + pinfo_count = readSymbol('ptlrpcds_num') + +- for idx in xrange(pinfo_count): ++ for idx in range(pinfo_count): + ptlrpcd = pinfo_rpcds[idx] +- for jdx in xrange(ptlrpcd.pd_nthreads): ++ for jdx in range(ptlrpcd.pd_nthreads): + pd = ptlrpcd.pd_threads[jdx] + callback(pd, *args) + pd = readSymbol('ptlrpcd_rcv') +@@ -227,8 +227,8 @@ def dump_daemon_rpclists(): + + def print_overview_entry(pd): + s = "%s:" % pd.pc_name +- print "%-14s ptlrpcd_ctl 0x%x ptlrpc_request_set 0x%x" % \ +- (s, Addr(pd), pd.pc_set) ++ print("%-14s ptlrpcd_ctl 0x%x ptlrpc_request_set 0x%x" % \ ++ (s, Addr(pd), pd.pc_set)) + + def dump_overview(): + foreach_ptlrpcd_ctl(print_overview_entry) +@@ -237,15 +237,15 @@ def print_pcset_stats(pd): + if pd.pc_set.set_new_count.counter != 0 or \ + pd.pc_set.set_remaining.counter != 0: + s = "%s:" %pd.pc_name +- print "%-13s 0x%-18x %-4d %-4d %-6d" % \ ++ print("%-13s 0x%-18x %-4d %-4d %-6d" % \ + (s, Addr(pd.pc_set), + pd.pc_set.set_refcount.counter, + pd.pc_set.set_new_count.counter, +- pd.pc_set.set_remaining.counter) ++ pd.pc_set.set_remaining.counter)) + + def dump_pcsets(): +- print '%-14s %-19s %-4s %-4s %-6s' % \ +- ("thread", "ptlrpc_request_set", "ref", "new", "remain") ++ print('%-14s %-19s %-4s %-4s %-6s' % \ ++ ("thread", "ptlrpc_request_set", "ref", "new", "remain")) + print_separator(52) + foreach_ptlrpcd_ctl(print_pcset_stats) + +diff --git a/contrib/debug_tools/epython_scripts/rpc_opcode.py b/contrib/debug_tools/epython_scripts/rpc_opcode.py +index 6aa8d582cf..0b06b09020 100644 +--- a/contrib/debug_tools/epython_scripts/rpc_opcode.py ++++ b/contrib/debug_tools/epython_scripts/rpc_opcode.py +@@ -114,9 +114,9 @@ opcodes = { + def translate_opcodes(opc_list): + for opc in opc_list: + try: +- print "o%d \t= %s" % (opc, opcodes[opc]) ++ print("o%d \t= %s" % (opc, opcodes[opc])) + except: +- print "o%d \t= unknown" % opc ++ print("o%d \t= unknown" % opc) + + + if __name__ == "__main__": +diff --git a/contrib/debug_tools/epython_scripts/rpc_stats.py b/contrib/debug_tools/epython_scripts/rpc_stats.py +index c271182154..f197fb15c3 100644 +--- a/contrib/debug_tools/epython_scripts/rpc_stats.py ++++ b/contrib/debug_tools/epython_scripts/rpc_stats.py +@@ -19,13 +19,13 @@ def get_cli_obd(client_obd): + cli = None + try: + cli = readSU('struct client_obd', client_obd) +- except Exception, e: ++ except Exception as e: + for dev in readSymbol('obd_devs'): + try: + if ll.obd2str(dev, 4) == client_obd: + cli = dev.u.cli + break +- except Exception, e: ++ except Exception as e: + continue + return cli + +@@ -40,16 +40,16 @@ def lprocfs_oh_sum(oh): + + def osc_rpc_stats_seq_show(client_obd): + if not client_obd: +- print "invalid input for field 'client_obd'" ++ print("invalid input for field 'client_obd'") + return 1 + cli = readSU('struct client_obd', client_obd) +- print "read RPCs in flight: %d" % cli.cl_r_in_flight +- print "write RPCs in flight: %d" % cli.cl_w_in_flight +- print "pending write pages: %d" % cli.cl_pending_w_pages.counter +- print "pending read pages: %d" % cli.cl_pending_r_pages.counter ++ print("read RPCs in flight: %d" % cli.cl_r_in_flight) ++ print("write RPCs in flight: %d" % cli.cl_w_in_flight) ++ print("pending write pages: %d" % cli.cl_pending_w_pages.counter) ++ print("pending read pages: %d" % cli.cl_pending_r_pages.counter) + +- print "\n\t\t\tread\t\t\twrite" +- print "pages per rpc rpcs % cum % | rpcs % cum %\n" ++ print("\n\t\t\tread\t\t\twrite") ++ print("pages per rpc rpcs % cum % | rpcs % cum %\n") + + read_tot = lprocfs_oh_sum(cli.cl_read_page_hist) + write_tot = lprocfs_oh_sum(cli.cl_write_page_hist) +@@ -62,16 +62,16 @@ def osc_rpc_stats_seq_show(client_obd): + + read_cum += r + write_cum += w +- print "%d:\t\t%10d %3d %3d | %10d %3d %3d" % \ ++ print("%d:\t\t%10d %3d %3d | %10d %3d %3d" % \ + (1 << i, r, pct(r, read_tot), + pct(read_cum, read_tot), w, + pct(w, write_tot), +- pct(write_cum, write_tot)) ++ pct(write_cum, write_tot))) + if read_cum == read_tot and write_cum == write_tot: + break + +- print "\n\t\t\tread\t\t\twrite" +- print "rpcs in flight rpcs % cum % | rpcs % cum %\n" ++ print("\n\t\t\tread\t\t\twrite") ++ print("rpcs in flight rpcs % cum % | rpcs % cum %\n") + + read_tot = lprocfs_oh_sum(cli.cl_read_rpc_hist) + write_tot = lprocfs_oh_sum(cli.cl_write_rpc_hist) +@@ -84,16 +84,16 @@ def osc_rpc_stats_seq_show(client_obd): + + read_cum += r + write_cum += w +- print "%d:\t\t%10d %3d %3d | %10d %3d %3d" % \ ++ print("%d:\t\t%10d %3d %3d | %10d %3d %3d" % \ + (i, r, pct(r, read_tot), + pct(read_cum, read_tot), w, + pct(w, write_tot), +- pct(write_cum, write_tot)) ++ pct(write_cum, write_tot))) + if read_cum == read_tot and write_cum == write_tot: + break + +- print "\n\t\t\tread\t\t\twrite" +- print "offset rpcs % cum % | rpcs % cum %\n" ++ print("\n\t\t\tread\t\t\twrite") ++ print("offset rpcs % cum % | rpcs % cum %\n") + + read_tot = lprocfs_oh_sum(cli.cl_read_offset_hist) + write_tot = lprocfs_oh_sum(cli.cl_write_offset_hist) +@@ -107,14 +107,14 @@ def osc_rpc_stats_seq_show(client_obd): + read_cum += r + write_cum += w + offset = 0 if i == 0 else 1 << (i - 1) +- print "%d: \t%10d %3d %3d | %10d %3d %3d" % \ ++ print("%d: \t%10d %3d %3d | %10d %3d %3d" % \ + (offset, r, pct(r, read_tot), + pct(read_cum, read_tot), w, + pct(w, write_tot), +- pct(write_cum, write_tot)) ++ pct(write_cum, write_tot))) + if read_cum == read_tot and write_cum == write_tot: + break +- print ++ print() + return 0 + + if __name__ == "__main__": +diff --git a/contrib/debug_tools/epython_scripts/sbi_ptrs.py b/contrib/debug_tools/epython_scripts/sbi_ptrs.py +index 78720393dd..c07be4948e 100644 +--- a/contrib/debug_tools/epython_scripts/sbi_ptrs.py ++++ b/contrib/debug_tools/epython_scripts/sbi_ptrs.py +@@ -16,11 +16,11 @@ def dump_inode(inode): + sb = readSU('struct super_block', inode.i_sb) + lsi = readSU('struct lustre_sb_info', sb.s_fs_info) + llsbi = readSU('struct ll_sb_info', lsi.lsi_llsbi) +- print "%x %x %x %x %x" % (Addr(inode), lli, sb, lsi, llsbi) ++ print("%x %x %x %x %x" % (Addr(inode), lli, sb, lsi, llsbi)) + + def dump_inode_list(inodes): +- print "%-16s %-16s %-16s %-16s %-16s" % ("inode", "ll_inode_info", +- "super_block", "lustre_sb_info", "ll_sb_info") ++ print("%-16s %-16s %-16s %-16s %-16s" % ("inode", "ll_inode_info", ++ "super_block", "lustre_sb_info", "ll_sb_info")) + for addr in inodes: + dump_inode(readSU('struct inode', addr)) + +diff --git a/contrib/debug_tools/epython_scripts/uniqueStacktrace.py b/contrib/debug_tools/epython_scripts/uniqueStacktrace.py +index 415aee16b4..c40d008bab 100644 +--- a/contrib/debug_tools/epython_scripts/uniqueStacktrace.py ++++ b/contrib/debug_tools/epython_scripts/uniqueStacktrace.py +@@ -72,13 +72,13 @@ def printRes(sort, printpid, printptr): + """ + for stack_trace, ptask_list in sort: + if printpid and not printptr: +- print "PID: %s" % (', '.join(p[0] for p in ptask_list)) ++ print("PID: %s" % (', '.join(p[0] for p in ptask_list))) + elif printpid and printptr: +- print "PID, TSK: %s" % (', '.join(p[0] + ': ' + p[1] for p in ptask_list)) ++ print("PID, TSK: %s" % (', '.join(p[0] + ': ' + p[1] for p in ptask_list))) + elif not printpid and printptr: +- print "TSK: %s" % (', '.join(p[1] for p in ptask_list)) +- print "TASKS: %d" %(len(ptask_list)) +- print "\t%s" %(stack_trace) ++ print("TSK: %s" % (', '.join(p[1] for p in ptask_list))) ++ print("TASKS: %d" %(len(ptask_list))) ++ print("\t%s" %(stack_trace)) + + + def main(): +-- +2.33.0 + diff --git a/0027-LU-15740-tests-scale-fs_log_size-by-OSTCOUNT.patch b/0027-LU-15740-tests-scale-fs_log_size-by-OSTCOUNT.patch deleted file mode 100644 index 50d501e0ce14df6904eee377bd61cb083f488b9d..0000000000000000000000000000000000000000 --- a/0027-LU-15740-tests-scale-fs_log_size-by-OSTCOUNT.patch +++ /dev/null @@ -1,188 +0,0 @@ -From 34e1409cad412086d509349f64dc9d77911d2fb8 Mon Sep 17 00:00:00 2001 -From: Andreas Dilger -Date: Fri, 24 Mar 2023 17:09:44 -0600 -Subject: [PATCH 27/61] LU-15740 tests: scale fs_log_size by OSTCOUNT - -The fs_log_size "free space skew" was being scaled by MDSCOUNT, -but in fact this parameter is only ever used to compare the OST -free space usage, so the OSTCOUNT should be used when scaling it. - -It is likely that the skew is actually caused by blocks allocated -by OST object directories and not llogs (no llogs used on OSTs for -many years), but it isn't worthwhile to rename the function. - -Lustre-change: https://review.whamcloud.com/50419 -Lustre-commit: fabec6f2cb39950a2f208567dac716e21880fa9f - -Test-Parameters: trivial testlist=runtests -Signed-off-by: Andreas Dilger -Change-Id: I97f05b10fa7ec367534b5bdce09feae5e93ebbe5 -Reviewed-by: Arshad Hussain -Reviewed-by: Alex Deiter -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51606 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: James Simmons -Reviewed-by: Oleg Drokin ---- - lustre/tests/replay-ost-single.sh | 20 ++++++++------------ - lustre/tests/replay-single.sh | 17 ++++++----------- - lustre/tests/test-framework.sh | 8 +++++--- - 3 files changed, 19 insertions(+), 26 deletions(-) - -diff --git a/lustre/tests/replay-ost-single.sh b/lustre/tests/replay-ost-single.sh -index e6ba4d0430..5ce0b7f28d 100755 ---- a/lustre/tests/replay-ost-single.sh -+++ b/lustre/tests/replay-ost-single.sh -@@ -169,10 +169,6 @@ test_5() { - } - run_test 5 "Fail OST during iozone" - --kbytesfree() { -- calc_osc_kbytes kbytesfree --} -- - test_6() { - remote_mds_nodsh && skip "remote MDS with nodsh" && return 0 - -@@ -185,7 +181,7 @@ test_6() { - wait_destroy_complete || error "first wait_destroy_complete failed" - sync_all_data - -- local before=$(kbytesfree) -+ local before=$(calc_osc_kbytes kbytesfree) - dd if=/dev/urandom bs=4096 count=1280 of=$f || error "dd failed" - $LFS getstripe $f || error "$LFS getstripe $f failed" - local stripe_index=$(lfs getstripe -i $f) -@@ -200,13 +196,13 @@ test_6() { - wait_mds_ost_sync || error "second wait_mds_ost_sync failed" - - # retry till statfs returns useful results -- local after_dd=$(kbytesfree) -+ local after_dd=$(calc_osc_kbytes kbytesfree) - local i=0 - while (( $before <= $after_dd && $i < 20 )); do - sync - sleep 1 - let ++i -- after_dd=$(kbytesfree) -+ after_dd=$(calc_osc_kbytes kbytesfree) - done - - log "before_free: $before after_dd_free: $after_dd took $i seconds" -@@ -222,7 +218,7 @@ test_6() { - # let the delete happen - wait_mds_ost_sync || error "third wait_mds_ost_sync failed" - wait_delete_completed || error "second wait_delete_completed failed" -- local after=$(kbytesfree) -+ local after=$(calc_osc_kbytes kbytesfree) - log "free_before: $before free_after: $after" - (( $before <= $after + $(fs_log_size) )) || - error "$before > $after + logsize $(fs_log_size)" -@@ -238,18 +234,18 @@ test_7() { - wait_mds_ost_sync || error "wait_mds_ost_sync failed" - wait_destroy_complete || error "wait_destroy_complete failed" - -- local before=$(kbytesfree) -+ local before=$(calc_osc_kbytes kbytesfree) - dd if=/dev/urandom bs=4096 count=1280 of=$f || - error "dd to file failed: $?" - - sync -- local after_dd=$(kbytesfree) -+ local after_dd=$(calc_osc_kbytes kbytesfree) - local i=0 - while (( $before <= $after_dd && $i < 10 )); do - sync - sleep 1 - let ++i -- after_dd=$(kbytesfree) -+ after_dd=$(calc_osc_kbytes kbytesfree) - done - - log "before: $before after_dd: $after_dd took $i seconds" -@@ -264,7 +260,7 @@ test_7() { - # let the delete happen - wait_mds_ost_sync || error "wait_mds_ost_sync failed" - wait_delete_completed || error "wait_delete_completed failed" -- local after=$(kbytesfree) -+ local after=$(calc_osc_kbytes kbytesfree) - log "before: $before after: $after" - (( $before <= $after + $(fs_log_size) )) || - error "$before > $after + logsize $(fs_log_size)" -diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh -index c6641c0d0e..c05570a850 100755 ---- a/lustre/tests/replay-single.sh -+++ b/lustre/tests/replay-single.sh -@@ -3289,23 +3289,17 @@ test_88() { #bug 17485 - } - run_test 88 "MDS should not assign same objid to different files " - --function calc_osc_kbytes_used() { -- local kbtotal=$(calc_osc_kbytes kbytestotal) -- local kbfree=$(calc_osc_kbytes kbytesfree) -- echo $((kbtotal-kbfree)) --} -- - test_89() { - cancel_lru_locks osc - mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed" - rm -f $DIR/$tdir/$tfile - wait_mds_ost_sync || error "initial MDS-OST sync timed out" - wait_delete_completed || error "initial wait delete timed out" -- local blocks1=$(calc_osc_kbytes_used) -+ local before=$(calc_osc_kbytes kbytesfree) - local write_size=$(fs_log_size) - - $LFS setstripe -i 0 -c 1 $DIR/$tdir/$tfile -- [ $write_size -lt 1024 ] && write_size=1024 -+ (( $write_size >= 1024 )) || write_size=1024 - dd if=/dev/zero bs=${write_size}k count=10 of=$DIR/$tdir/$tfile - sync - stop ost1 -@@ -3323,10 +3317,11 @@ test_89() { - - wait_mds_ost_sync || error "MDS-OST sync timed out" - wait_delete_completed || error "wait delete timed out" -- local blocks2=$(calc_osc_kbytes_used) -+ local after=$(calc_osc_kbytes kbytesfree) - -- [ $((blocks2 - blocks1)) -le $(fs_log_size) ] || -- error $((blocks2 - blocks1)) blocks leaked -+ log "free_before: $before free_after: $after" -+ (( $before <= $after + $(fs_log_size) )) || -+ error "kbytesfree $before > $after + margin $(fs_log_size)" - } - run_test 89 "no disk space leak on late ost connection" - -diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh -index 4d556d05bd..825b7189d7 100755 ---- a/lustre/tests/test-framework.sh -+++ b/lustre/tests/test-framework.sh -@@ -918,18 +918,20 @@ unload_modules() { - } - - fs_log_size() { -- local facet=${1:-$SINGLEMDS} -+ local facet=${1:-ost1} - local size=0 -+ local mult=$OSTCOUNT - - case $(facet_fstype $facet) in -- ldiskfs) size=72;; # largest seen is 64, leave some headroom -+ ldiskfs) size=32;; # largest seen is 64 with multiple OSTs - # grant_block_size is in bytes, allow at least 2x max blocksize - zfs) size=$(lctl get_param osc.$FSNAME*.import | - awk '/grant_block_size:/ {print $2/512; exit;}') - ;; - esac - -- echo -n $((size * MDSCOUNT)) -+ [[ $facet =~ mds ]] && mult=$MDTCOUNT -+ echo -n $((size * mult)) - } - - fs_inode_ksize() { --- -2.33.0 - diff --git a/0038-LU-16322-build-Add-client-build-support-for-openEule.patch b/0028-LU-16322-build-Add-client-build-support-for-openEule.patch similarity index 91% rename from 0038-LU-16322-build-Add-client-build-support-for-openEule.patch rename to 0028-LU-16322-build-Add-client-build-support-for-openEule.patch index 31340c5079ce1f4810c550cb204bc89fd5fc6029..d80adc72250b1009323d27b1b0e7c3f43a6f7612 100644 --- a/0038-LU-16322-build-Add-client-build-support-for-openEule.patch +++ b/0028-LU-16322-build-Add-client-build-support-for-openEule.patch @@ -1,7 +1,7 @@ -From 1efdc0f82bf67c06ad50d09c24cab97a7f0e2e89 Mon Sep 17 00:00:00 2001 +From 4eacd4c63384cc10580036ac50758855db1f475c Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Wed, 26 Oct 2022 08:58:14 +0000 -Subject: [PATCH 38/61] LU-16322: build: Add client build support for openEuler +Subject: [PATCH 28/49] LU-16322: build: Add client build support for openEuler The kernel of current openEuler LTS version 22.03 is based on Linux 5.10.0 which is already supported in Lustre master. Thus we only need @@ -93,7 +93,7 @@ index db1b2ea86a..63e212a3d1 100644 IN_KERNEL="${PACKAGE}"]) AC_MSG_RESULT($KMP_MODDIR) diff --git a/contrib/lbuild/funcs.sh b/contrib/lbuild/funcs.sh -index a6c39473cf..78e2af9468 100644 +index 29cb66bdd9..25978095d4 100644 --- a/contrib/lbuild/funcs.sh +++ b/contrib/lbuild/funcs.sh @@ -154,6 +154,11 @@ autodetect_distro() { @@ -108,9 +108,9 @@ index a6c39473cf..78e2af9468 100644 *) fatal 1 "I don't know what distro name $name and version $version is.\nEither update autodetect_distro() or use the --distro argument." ;; -@@ -207,6 +212,7 @@ autodetect_target() { - sles15.3) target="$(uname -r | cut -d . -f 1,2)-sles15sp3";; +@@ -209,6 +214,7 @@ autodetect_target() { sles15.4) target="$(uname -r | cut -d . -f 1,2)-sles15sp4";; + sles15.5) target="$(uname -r | cut -d . -f 1,2)-sles15sp5";; fc18) target="3.x-fc18";; + oe2203) target="5.10-oe2203";; *) fatal 1 "I don't know what distro $distro is.\nEither update autodetect_target() or use the --target argument.";; @@ -176,10 +176,10 @@ index 0000000000..2abdcb6c52 + fi +} diff --git a/lustre.spec.in b/lustre.spec.in -index f4ac0c894d..a8c0934428 100644 +index 2678d608db..1ddb1f9224 100644 --- a/lustre.spec.in +++ b/lustre.spec.in -@@ -136,7 +136,7 @@ +@@ -137,7 +137,7 @@ # requires want to set a version including epoch %global krequires %(echo %{kver} | sed -e 's/\.x86_64$//' -e 's/\.i[3456]86$//' -e 's/-smp$//' -e 's/-bigsmp$//' -e 's/[-.]ppc64$//' -e 's/\.aarch64$//' -e 's/-default$//' -e 's/-%{_flavor}//') @@ -188,7 +188,7 @@ index f4ac0c894d..a8c0934428 100644 %global requires_kmod_name kmod-%{lustre_name} %global requires_kmod_osd_zfs_name kmod-%{lustre_name}-osd-zfs %if %{with lustre_tests} -@@ -175,6 +175,11 @@ +@@ -176,6 +176,11 @@ %define with_systemd 1 %endif @@ -200,7 +200,7 @@ index f4ac0c894d..a8c0934428 100644 %{!?mpi_name: %global mpi_name openmpi} Summary: Lustre File System -@@ -207,6 +212,9 @@ BuildRequires: libtool libyaml-devel zlib-devel libnl3-devel flex bison +@@ -208,6 +213,9 @@ BuildRequires: libtool libyaml-devel zlib-devel libnl3-devel flex bison BuildRequires: redhat-rpm-config BuildRequires: pkgconfig %else @@ -210,7 +210,7 @@ index f4ac0c894d..a8c0934428 100644 BuildRequires: pkg-config %endif %if %{with gss} -@@ -220,20 +228,20 @@ Provides: lustre-server = %{version}-%{release} +@@ -221,7 +229,7 @@ Provides: lustre-server = %{version}-%{release} %endif Obsoletes: lustre-client < %{version} Provides: lustre-client = %{version}-%{release} @@ -219,6 +219,7 @@ index f4ac0c894d..a8c0934428 100644 #suse don't support selinux BuildRequires: libselinux-devel %endif +@@ -231,13 +239,13 @@ BuildRequires: pkgconfig(libselinux) openssl-devel %if %{with lustre_modules} %if %{with mofed} BuildRequires: mlnx-ofa_kernel-devel @@ -234,7 +235,7 @@ index f4ac0c894d..a8c0934428 100644 BuildRequires: kernel-rpm-macros %endif BuildRequires: %kernel_module_package_buildreqs -@@ -374,7 +382,7 @@ Requires: attr, rsync, perl, lsof, /usr/bin/getconf +@@ -378,7 +386,7 @@ Requires: attr, rsync, perl, lsof, /usr/bin/getconf BuildRequires: mpich-devel %endif %if "%{mpi_name}" == "openmpi" @@ -243,7 +244,7 @@ index f4ac0c894d..a8c0934428 100644 BuildRequires: openmpi-devel %else BuildRequires: openmpi2-devel -@@ -626,7 +634,7 @@ echo '%{_sysconfdir}/ha.d/resource.d/Lustre' >>lustre.files +@@ -631,7 +639,7 @@ echo '%{_sysconfdir}/ha.d/resource.d/Lustre' >>lustre.files echo '%{_unitdir}/lnet.service' >>lustre.files %endif @@ -253,10 +254,10 @@ index f4ac0c894d..a8c0934428 100644 %if %{with servers} echo '%{_sysconfdir}/init.d/lustre' >>lustre.files diff --git a/lustre/ChangeLog b/lustre/ChangeLog -index 4b21088869..192662875e 100644 +index 782557e3f5..e028e29051 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog -@@ -206,6 +206,7 @@ TBD Whamcloud +@@ -210,6 +210,7 @@ 5.8.0-53 (Ubuntu 20.04.2 HWE) 5.11.0-31 (Ubuntu 20.04.3 HWE) 5.11.0 (vanilla kernel.org) @@ -265,12 +266,12 @@ index 4b21088869..192662875e 100644 * Recommended ZFS version: 2.1.5 * NFS export disabled when stack size < 8192 (32-bit Lustre clients), diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 -index 2927df790c..12afe95fbc 100644 +index dd1df4950e..a88ccc6251 100644 --- a/lustre/autoconf/lustre-core.m4 +++ b/lustre/autoconf/lustre-core.m4 -@@ -3496,6 +3496,7 @@ lustre/kernel_patches/targets/5.3-sles15sp2.target - lustre/kernel_patches/targets/5.3-sles15sp3.target +@@ -4474,6 +4474,7 @@ lustre/kernel_patches/targets/5.3-sles15sp3.target lustre/kernel_patches/targets/5.14-sles15sp4.target + lustre/kernel_patches/targets/5.14-sles15sp5.target lustre/kernel_patches/targets/3.x-fc18.target +lustre/kernel_patches/targets/5.10-oe2203.target lustre/ldlm/Makefile diff --git a/0028-LU-16943-tests-fix-replay-single-135-under-hard-fail.patch b/0028-LU-16943-tests-fix-replay-single-135-under-hard-fail.patch deleted file mode 100644 index f0b0f7c1bef4bd2a20c1662f829ae85631f99fdd..0000000000000000000000000000000000000000 --- a/0028-LU-16943-tests-fix-replay-single-135-under-hard-fail.patch +++ /dev/null @@ -1,84 +0,0 @@ -From dc943357276b3bec99d24c89d9b36f7b3cf20aeb Mon Sep 17 00:00:00 2001 -From: Jian Yu -Date: Fri, 14 Jul 2023 14:04:42 +0800 -Subject: [PATCH 28/61] LU-16943 tests: fix replay-single/135 under hard - failure mode - -This patch fixes replay-single test_135() to load libcfs module -on the failover partner node to avoid 'fail_val' setting error. -It also fixes the issue that not all of the OSTs are mounted after -failing back ost1. - -Lustre-change: https://review.whamcloud.com/51574 -Lustre-commit: 74140e5df4c094f7f0e923e1b82c464b18e8a7cc - -Test-Parameters: trivial testlist=replay-single -Test-Parameters: trivial fstype=zfs testlist=replay-single - -Test-Parameters: trivial env=FAILURE_MODE=HARD \ - clientcount=4 mdtcount=1 mdscount=2 osscount=2 \ - austeroptions=-R failover=true iscsi=1 \ - testlist=replay-single - -Change-Id: Id46c722a6db9d832829a739f41f7462b32a6d9d9 -Signed-off-by: Jian Yu -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51608 -Reviewed-by: Oleg Drokin -Reviewed-by: Alex Deiter -Tested-by: jenkins -Tested-by: Maloo ---- - lustre/tests/replay-single.sh | 7 +++++++ - lustre/tests/test-framework.sh | 9 +++++++++ - 2 files changed, 16 insertions(+) - -diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh -index c05570a850..f631b73125 100755 ---- a/lustre/tests/replay-single.sh -+++ b/lustre/tests/replay-single.sh -@@ -4977,6 +4977,8 @@ test_135() { - - #define OBD_FAIL_TGT_REPLAY_RECONNECT 0x32d - # Make sure lock replay server side never completes and errors out. -+ do_rpc_nodes $(facet_active_host ost1) \ -+ load_module ../libcfs/libcfs/libcfs - do_facet ost1 "$LCTL set_param fail_val=20" - do_facet ost1 "$LCTL set_param fail_loc=0x32d" - -@@ -4993,8 +4995,13 @@ test_135() { - change_active ost1 - wait_for_facet ost1 - -+ do_rpc_nodes $(facet_active_host ost1) \ -+ load_module ../libcfs/libcfs/libcfs - do_facet ost1 "$LCTL set_param fail_loc=0" - mount_facet ost1 -+ unmountoss -+ mountoss -+ clients_up || clients_up || error "$LFS df $MOUNT failed" - echo blah > $DIR/$tdir/file.test2 - - rm -rf $DIR/$tdir -diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh -index 825b7189d7..2c92993117 100755 ---- a/lustre/tests/test-framework.sh -+++ b/lustre/tests/test-framework.sh -@@ -5033,6 +5033,15 @@ mountmds() { - done - } - -+unmountoss() { -+ local num -+ -+ for num in $(seq $OSTCOUNT); do -+ stop ost$num -f -+ rm -f $TMP/ost${num}active -+ done -+} -+ - mountoss() { - local num - local devname --- -2.33.0 - diff --git a/0039-LU-16481-build-add-server-support-for-openEuler.patch b/0029-LU-16481-build-add-server-support-for-openEuler.patch similarity index 99% rename from 0039-LU-16481-build-add-server-support-for-openEuler.patch rename to 0029-LU-16481-build-add-server-support-for-openEuler.patch index b1da3fde7f3ee16d3f5f37f7deeb72023e9da73d..f486ab629f30e9ec1c9c7804918cc15f11582909 100644 --- a/0039-LU-16481-build-add-server-support-for-openEuler.patch +++ b/0029-LU-16481-build-add-server-support-for-openEuler.patch @@ -1,7 +1,7 @@ -From f989b37b3bd1e4f7a465e959eaf7559438bab248 Mon Sep 17 00:00:00 2001 +From b9031b4d2432e11a8f65b66c25fed8974f6f021d Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Mon, 21 Nov 2022 03:36:38 +0000 -Subject: [PATCH 39/61] LU-16481 build: add server support for openEuler +Subject: [PATCH 29/49] LU-16481 build: add server support for openEuler openEuer uses dnf as rpm pkg manager, it is somewhat like RHEL/Fedora. The current openEuler LTS 22.03 kernel is based on Linux 5.10.0. @@ -83,10 +83,10 @@ Signed-off-by: Xinliang Liu create mode 100644 ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203.series diff --git a/config/lustre-build-ldiskfs.m4 b/config/lustre-build-ldiskfs.m4 -index 14ad1e2215..8bc334b951 100644 +index 2e0b3e6d76..9ece25f757 100644 --- a/config/lustre-build-ldiskfs.m4 +++ b/config/lustre-build-ldiskfs.m4 -@@ -123,9 +123,13 @@ AS_IF([test x$RHEL_KERNEL = xyes], [ +@@ -124,9 +124,13 @@ AS_IF([test x$RHEL_KERNEL = xyes], [ [LDISKFS_SERIES="5.8.0-53-ubuntu20.series"], [LDISKFS_SERIES="5.8.0-63-ubuntu20.series"], [LDISKFS_SERIES="5.8.0-ml.series"]) @@ -131,10 +131,10 @@ index 63e212a3d1..1ff6c47856 100644 ]) diff --git a/config/lustre-build.m4 b/config/lustre-build.m4 -index 28eb3ecfe8..0e775d9d41 100644 +index 3d80397a34..a7baedeb3f 100644 --- a/config/lustre-build.m4 +++ b/config/lustre-build.m4 -@@ -448,7 +448,7 @@ AM_CONDITIONAL([DOC], [test x$ENABLE_DOC = x1]) +@@ -452,7 +452,7 @@ AM_CONDITIONAL([DOC], [test x$ENABLE_DOC = x1]) AM_CONDITIONAL([MANPAGES], [test x$enable_manpages = xyes]) AM_CONDITIONAL([LINUX], [test x$lb_target_os = xlinux]) AM_CONDITIONAL([USE_QUILT], [test x$use_quilt = xyes]) @@ -2027,10 +2027,10 @@ index 0000000000..65d1f3a312 +oe2203/ext4-delayed-iput.patch +rhel8/ext4-old_ea_inodes_handling_fix.patch diff --git a/lustre/ChangeLog b/lustre/ChangeLog -index 192662875e..9401e95187 100644 +index e028e29051..d58aa724bc 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog -@@ -100,6 +100,7 @@ TBD Whamcloud +@@ -104,6 +104,7 @@ vanilla linux 5.4.0 (ZFS + ldiskfs) vanilla linux 5.4.21 (ZFS + ldiskfs) vanilla linux 5.4.136 (ZFS + ldiskfs) @@ -2038,7 +2038,7 @@ index 192662875e..9401e95187 100644 * ldiskfs needs an ldiskfs patch series for that kernel, ZFS does not * Client primary kernels built and tested during release cycle: 5.14.0-284.11.1.el9 (RHEL9.2) -@@ -206,7 +207,7 @@ TBD Whamcloud +@@ -210,7 +211,7 @@ 5.8.0-53 (Ubuntu 20.04.2 HWE) 5.11.0-31 (Ubuntu 20.04.3 HWE) 5.11.0 (vanilla kernel.org) diff --git a/0029-LU-16517-build-pass-extra-configure-options-to-make-.patch b/0029-LU-16517-build-pass-extra-configure-options-to-make-.patch deleted file mode 100644 index d181bf3bee9ab0d11a2093c5ffa28d47c0e04a36..0000000000000000000000000000000000000000 --- a/0029-LU-16517-build-pass-extra-configure-options-to-make-.patch +++ /dev/null @@ -1,514 +0,0 @@ -From 5abc979eb64d8a78888aea63c0cf7e285fa0a4dd Mon Sep 17 00:00:00 2001 -From: Jian Yu -Date: Tue, 30 May 2023 23:40:09 -0700 -Subject: [PATCH 29/61] LU-16517 build: pass extra configure options to "make - debs" - -While running "make debs", the configure command in debian/rules -ignores some user defined configure options. This patch fixes -the issue by adding the detection of the extra options into -debian/rules. - -Lustre-change: https://review.whamcloud.com/50464 -Lustre-commit: 3989529f22f5c54a98e445674b4b3cc443a3af5f - -Test-Parameters: trivial clientdistro=ubuntu2004 - -Change-Id: Ia9db4e05abf33834cb3c853f4f0829dadc8d7400 -Signed-off-by: Jian Yu -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51178 -Reviewed-by: Andreas Dilger -Reviewed-by: Alex Deiter -Reviewed-by: Oleg Drokin -Tested-by: Maloo -Tested-by: jenkins ---- - autoMakefile.am | 95 ++++++++++++++++++++++++++++++++ - debian/rules | 39 ++++++++----- - libcfs/autoconf/lustre-libcfs.m4 | 39 ++++++++++--- - lnet/autoconf/lustre-lnet.m4 | 12 ++++ - lustre/autoconf/lustre-core.m4 | 87 +++++++++++++++++++++-------- - lustre/ldlm/ldlm_lib.c | 2 +- - lustre/llite/llite_lib.c | 4 +- - 7 files changed, 228 insertions(+), 50 deletions(-) - -diff --git a/autoMakefile.am b/autoMakefile.am -index 8c2eeb7f4c..8a2cd04953 100644 ---- a/autoMakefile.am -+++ b/autoMakefile.am -@@ -302,6 +302,101 @@ debs: undef.h debs_common - if test "x@systemdsystemunitdir@" != "x"; then \ - export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} systemd"; \ - fi; \ -+ if test "x@ENABLE_PINGER@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} pinger"; \ -+ elif test "x@ENABLE_PINGER@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nopinger"; \ -+ fi; \ -+ if test "x@ENABLE_CHECKSUM@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} checksum"; \ -+ elif test "x@ENABLE_CHECKSUM@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nochecksum"; \ -+ fi; \ -+ if test "x@ENABLE_FLOCK@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} flock"; \ -+ elif test "x@ENABLE_FLOCK@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} noflock"; \ -+ fi; \ -+ if test "x@ENABLE_HEALTH_WRITE@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} health_write"; \ -+ elif test "x@ENABLE_HEALTH_WRITE@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nohealth_write"; \ -+ fi; \ -+ if test "x@ENABLE_LRU_RESIZE@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} lru-resize"; \ -+ elif test "x@ENABLE_LRU_RESIZE@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nolru-resize"; \ -+ fi; \ -+ if test "x@ENABLE_MINDF@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} mindf"; \ -+ elif test "x@ENABLE_MINDF@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nomindf"; \ -+ fi; \ -+ if test "x@ENABLE_FAIL_ALLOC@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} fail-alloc"; \ -+ elif test "x@ENABLE_FAIL_ALLOC@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nofail-alloc"; \ -+ fi; \ -+ if test "x@ENABLE_INVARIANTS@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} invariants"; \ -+ elif test "x@ENABLE_INVARIANTS@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} noinvariants"; \ -+ fi; \ -+ if test "x@ENABLE_LU_REF@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} lu_ref"; \ -+ elif test "x@ENABLE_LU_REF@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nolu_ref"; \ -+ fi; \ -+ if test "x@ENABLE_PGSTAT_TRACK@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} pgstate-track"; \ -+ elif test "x@ENABLE_PGSTAT_TRACK@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nopgstate-track"; \ -+ fi; \ -+ if test "x@ENABLE_LIBCFS_CDEBUG@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} libcfs-cdebug"; \ -+ elif test "x@ENABLE_LIBCFS_CDEBUG@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nolibcfs-cdebug"; \ -+ fi; \ -+ if test "x@ENABLE_LIBCFS_TRACE@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} libcfs-trace"; \ -+ elif test "x@ENABLE_LIBCFS_TRACE@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nolibcfs-trace"; \ -+ fi; \ -+ if test "x@ENABLE_LIBCFS_ASSERT@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} libcfs-assert"; \ -+ elif test "x@ENABLE_LIBCFS_ASSERT@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nolibcfs-assert"; \ -+ fi; \ -+ if test "x@ENABLE_PANIC_DUMPLOG@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} panic_dumplog"; \ -+ elif test "x@ENABLE_PANIC_DUMPLOG@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nopanic_dumplog"; \ -+ fi; \ -+ if test "x@ENABLE_READLINE@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} readline"; \ -+ elif test "x@ENABLE_READLINE@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} noreadline"; \ -+ fi; \ -+ if test "x@ENABLE_LIBPTHREAD@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} libpthread"; \ -+ elif test "x@ENABLE_LIBPTHREAD@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nolibpthread"; \ -+ fi; \ -+ if test "x@ENABLE_BACKOFF@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} backoff"; \ -+ elif test "x@ENABLE_BACKOFF@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nobackoff"; \ -+ fi; \ -+ if test "x@ENABLE_GNI@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} gni"; \ -+ elif test "x@ENABLE_GNI@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} nogni"; \ -+ fi; \ -+ if test "x@ENABLE_EFENCE@" = "xyes"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} efence"; \ -+ elif test "x@ENABLE_EFENCE@" = "xno"; then \ -+ export DEB_BUILD_PROFILES="$${DEB_BUILD_PROFILES} noefence"; \ -+ fi; \ - export KERNEL_OBJ="$(LINUX_OBJ)"; \ - export KERNEL_SRC="$(LINUX)"; \ - echo "Enabled Build Profiles: $${DEB_BUILD_PROFILES}"; \ -diff --git a/debian/rules b/debian/rules -index 074b59ac39..344f255c04 100755 ---- a/debian/rules -+++ b/debian/rules -@@ -189,16 +189,18 @@ configure-stamp: autogen-stamp debian/control.main debian/control.modules.in - if echo "$${DEB_BUILD_PROFILES}" | grep -q "o2ib"; then \ - export EXTRAFLAGS="$${EXTRAFLAGS} --with-o2ib=$${O2IB_SRC}"; \ - fi; \ -- if echo "$${DEB_BUILD_PROFILES}" | grep -qw "gss"; then \ -- export EXTRAFLAGS="$${EXTRAFLAGS} --enable-gss"; \ -- elif echo "$${DEB_BUILD_PROFILES}" | grep -qw "nogss"; then \ -- export EXTRAFLAGS="$${EXTRAFLAGS} --disable-gss"; \ -- fi; \ -- if echo "$${DEB_BUILD_PROFILES}" | grep -qw "crypto"; then \ -- export EXTRAFLAGS="$${EXTRAFLAGS} --enable-crypto"; \ -- elif echo "$${DEB_BUILD_PROFILES}" | grep -qw "nocrypto"; then \ -- export EXTRAFLAGS="$${EXTRAFLAGS} --disable-crypto"; \ -- fi; \ -+ options="gss crypto pinger checksum flock health_write lru-resize"; \ -+ options="$${options} mindf fail-alloc invariants lu_ref pgstate-track"; \ -+ options="$${options} libcfs-cdebug libcfs-trace libcfs-assert"; \ -+ options="$${options} panic_dumplog readline libpthread"; \ -+ options="$${options} backoff gni efence"; \ -+ for option in $${options}; do \ -+ if echo "$${DEB_BUILD_PROFILES}" | grep -qw "$${option}"; then \ -+ export EXTRAFLAGS="$${EXTRAFLAGS} --enable-$${option}"; \ -+ elif echo "$${DEB_BUILD_PROFILES}" | grep -qw "no$${option}"; then \ -+ export EXTRAFLAGS="$${EXTRAFLAGS} --disable-$${option}"; \ -+ fi; \ -+ done; \ - if [ -f "$${CONFIG_CACHE_FILE}" ]; then \ - export TMP_CACHE_FILE=$$(mktemp); \ - sed -e "/ac_cv_env/d" \ -@@ -490,11 +492,18 @@ kdist_config: prep-deb-files patch-stamp - if echo "$${DEB_BUILD_PROFILES}" | grep -q "o2ib"; then \ - export EXTRAFLAGS="$${EXTRAFLAGS} --with-o2ib=$${O2IB_SRC}"; \ - fi; \ -- if echo "$${DEB_BUILD_PROFILES}" | grep -qw "crypto"; then \ -- export EXTRAFLAGS="$${EXTRAFLAGS} --enable-crypto"; \ -- elif echo "$${DEB_BUILD_PROFILES}" | grep -qw "nocrypto"; then \ -- export EXTRAFLAGS="$${EXTRAFLAGS} --disable-crypto"; \ -- fi; \ -+ options="gss crypto pinger checksum flock health_write lru-resize"; \ -+ options="$${options} mindf fail-alloc invariants lu_ref pgstate-track"; \ -+ options="$${options} libcfs-cdebug libcfs-trace libcfs-assert"; \ -+ options="$${options} panic_dumplog readline libpthread"; \ -+ options="$${options} backoff gni efence"; \ -+ for option in $${options}; do \ -+ if echo "$${DEB_BUILD_PROFILES}" | grep -qw "$${option}"; then \ -+ export EXTRAFLAGS="$${EXTRAFLAGS} --enable-$${option}"; \ -+ elif echo "$${DEB_BUILD_PROFILES}" | grep -qw "no$${option}"; then \ -+ export EXTRAFLAGS="$${EXTRAFLAGS} --disable-$${option}"; \ -+ fi; \ -+ done; \ - if [ -f "$${CONFIG_CACHE_FILE}" ]; then \ - export TMP_CACHE_FILE=$$(mktemp --tmpdir newconfig-XXXXXXXX.cache); \ - sed -e "/ac_cv_env/d" \ -diff --git a/libcfs/autoconf/lustre-libcfs.m4 b/libcfs/autoconf/lustre-libcfs.m4 -index 66c97eb044..2e99cff179 100644 ---- a/libcfs/autoconf/lustre-libcfs.m4 -+++ b/libcfs/autoconf/lustre-libcfs.m4 -@@ -10,8 +10,12 @@ AC_ARG_ENABLE([libcfs_cdebug], - [disable libcfs CDEBUG, CWARN]), - [], [enable_libcfs_cdebug="yes"]) - AC_MSG_RESULT([$enable_libcfs_cdebug]) --AS_IF([test "x$enable_libcfs_cdebug" = xyes], -- [AC_DEFINE(CDEBUG_ENABLED, 1, [enable libcfs CDEBUG, CWARN])]) -+AS_IF([test "x$enable_libcfs_cdebug" = xyes], [ -+ AC_DEFINE(CDEBUG_ENABLED, 1, [enable libcfs CDEBUG, CWARN]) -+ AC_SUBST(ENABLE_LIBCFS_CDEBUG, yes) -+], [ -+ AC_SUBST(ENABLE_LIBCFS_CDEBUG, no) -+]) - - AC_MSG_CHECKING([whether to enable ENTRY/EXIT]) - AC_ARG_ENABLE([libcfs_trace], -@@ -19,8 +23,12 @@ AC_ARG_ENABLE([libcfs_trace], - [disable libcfs ENTRY/EXIT]), - [], [enable_libcfs_trace="yes"]) - AC_MSG_RESULT([$enable_libcfs_trace]) --AS_IF([test "x$enable_libcfs_trace" = xyes], -- [AC_DEFINE(CDEBUG_ENTRY_EXIT, 1, [enable libcfs ENTRY/EXIT])]) -+AS_IF([test "x$enable_libcfs_trace" = xyes], [ -+ AC_DEFINE(CDEBUG_ENTRY_EXIT, 1, [enable libcfs ENTRY/EXIT]) -+ AC_SUBST(ENABLE_LIBCFS_TRACE, yes) -+], [ -+ AC_SUBST(ENABLE_LIBCFS_TRACE, no) -+]) - - AC_MSG_CHECKING([whether to enable LASSERT, LASSERTF]) - AC_ARG_ENABLE([libcfs_assert], -@@ -28,8 +36,12 @@ AC_ARG_ENABLE([libcfs_assert], - [disable libcfs LASSERT, LASSERTF]), - [], [enable_libcfs_assert="yes"]) - AC_MSG_RESULT([$enable_libcfs_assert]) --AS_IF([test x$enable_libcfs_assert = xyes], -- [AC_DEFINE(LIBCFS_DEBUG, 1, [enable libcfs LASSERT, LASSERTF])]) -+AS_IF([test x$enable_libcfs_assert = xyes], [ -+ AC_DEFINE(LIBCFS_DEBUG, 1, [enable libcfs LASSERT, LASSERTF]) -+ AC_SUBST(ENABLE_LIBCFS_ASSERT, yes) -+], [ -+ AC_SUBST(ENABLE_LIBCFS_ASSERT, no) -+]) - ]) # LIBCFS_CONFIG_CDEBUG - - # -@@ -44,8 +56,12 @@ AC_ARG_ENABLE([panic_dumplog], - [enable panic_dumplog]), - [], [enable_panic_dumplog="no"]) - AC_MSG_RESULT([$enable_panic_dumplog]) --AS_IF([test "x$enable_panic_dumplog" = xyes], -- [AC_DEFINE(LNET_DUMP_ON_PANIC, 1, [use dumplog on panic])]) -+AS_IF([test "x$enable_panic_dumplog" = xyes], [ -+ AC_DEFINE(LNET_DUMP_ON_PANIC, 1, [use dumplog on panic]) -+ AC_SUBST(ENABLE_PANIC_DUMPLOG, yes) -+], [ -+ AC_SUBST(ENABLE_PANIC_DUMPLOG, no) -+]) - ]) # LIBCFS_CONFIG_PANIC_DUMPLOG - - # -@@ -2534,6 +2550,10 @@ AS_IF([test "x$enable_readline" = xyes], [ - AC_DEFINE(HAVE_LIBREADLINE, 1, - [readline library is available]) - ]) -+ -+ AC_SUBST(ENABLE_READLINE, yes) -+], [ -+ AC_SUBST(ENABLE_READLINE, no) - ]) - AC_SUBST(LIBREADLINE) - -@@ -2551,7 +2571,10 @@ AS_IF([test "x$enable_libpthread" = xyes], [ - AC_DEFINE([HAVE_LIBPTHREAD], 1, - [use libpthread for libcfs library]) - ]) -+ -+ AC_SUBST(ENABLE_LIBPTHREAD, yes) - ], [ -+ AC_SUBST(ENABLE_LIBPTHREAD, no) - AC_MSG_WARN([Using libpthread for libcfs library is disabled explicitly]) - ]) - AC_SUBST(PTHREAD_LIBS) -diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 -index 05dd808a6a..da6fd582b2 100644 ---- a/lnet/autoconf/lustre-lnet.m4 -+++ b/lnet/autoconf/lustre-lnet.m4 -@@ -20,6 +20,11 @@ AS_IF([test "x$enable_backoff" = xyes], [ - ], [ - AC_MSG_RESULT([no]) - ]) -+ -+ AC_SUBST(ENABLE_BACKOFF, yes) -+], [ -+ AC_SUBST(ENABLE_BACKOFF, no) -+ - ]) - ]) # LN_CONFIG_BACKOFF - -@@ -681,6 +686,10 @@ AS_IF([test "x$enable_gni" = xyes], [ - GNICPPFLAGS="$GNICPPFLAGS -DGNILND_USE_RCA=1" - ]) - EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" -+ -+ AC_SUBST(ENABLE_GNI, yes) -+], [ -+ AC_SUBST(ENABLE_GNI, no) - ]) - AC_SUBST(GNICPPFLAGS) - AC_SUBST(GNILND) -@@ -1020,6 +1029,9 @@ AS_IF([test "$enable_efence" = yes], [ - LIBEFENCE="-lefence" - AC_DEFINE(HAVE_LIBEFENCE, 1, - [libefence support is requested]) -+ AC_SUBST(ENABLE_EFENCE, yes) -+], [ -+ AC_SUBST(ENABLE_EFENCE, no) - ]) - AC_SUBST(LIBEFENCE) - -diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 -index 524645f708..2927df790c 100644 ---- a/lustre/autoconf/lustre-core.m4 -+++ b/lustre/autoconf/lustre-core.m4 -@@ -137,8 +137,12 @@ AC_ARG_ENABLE([pinger], - [disable recovery pinger support]), - [], [enable_pinger="yes"]) - AC_MSG_RESULT([$enable_pinger]) --AS_IF([test "x$enable_pinger" != xno], -- [AC_DEFINE(CONFIG_LUSTRE_FS_PINGER, 1,[Use the Pinger])]) -+AS_IF([test "x$enable_pinger" != xno], [ -+ AC_DEFINE(CONFIG_LUSTRE_FS_PINGER, 1, [Use the Pinger]) -+ AC_SUBST(ENABLE_PINGER, yes) -+], [ -+ AC_SUBST(ENABLE_PINGER, no) -+]) - ]) # LC_CONFIG_PINGER - - # -@@ -153,8 +157,12 @@ AC_ARG_ENABLE([checksum], - [disable data checksum support]), - [], [enable_checksum="yes"]) - AC_MSG_RESULT([$enable_checksum]) --AS_IF([test "x$enable_checksum" != xno], -- [AC_DEFINE(ENABLE_CHECKSUM, 1, [do data checksums])]) -+AS_IF([test "x$enable_checksum" != xno], [ -+ AC_DEFINE(CONFIG_ENABLE_CHECKSUM, 1, [do data checksums]) -+ AC_SUBST(ENABLE_CHECKSUM, yes) -+], [ -+ AC_SUBST(ENABLE_CHECKSUM, no) -+]) - ]) # LC_CONFIG_CHECKSUM - - # -@@ -169,8 +177,12 @@ AC_ARG_ENABLE([flock], - [disable flock by default]), - [], [enable_flock="yes"]) - AC_MSG_RESULT([$enable_flock]) --AS_IF([test "x$enable_flock" != xno], -- [AC_DEFINE(ENABLE_FLOCK, 1, [enable flock by default])]) -+AS_IF([test "x$enable_flock" != xno], [ -+ AC_DEFINE(CONFIG_ENABLE_FLOCK, 1, [enable flock by default]) -+ AC_SUBST(ENABLE_FLOCK, yes) -+], [ -+ AC_SUBST(ENABLE_FLOCK, no) -+]) - ]) # LC_CONFIG_FLOCK - - # -@@ -185,8 +197,12 @@ AC_ARG_ENABLE([health_write], - [enable disk writes when doing health check]), - [], [enable_health_write="no"]) - AC_MSG_RESULT([$enable_health_write]) --AS_IF([test "x$enable_health_write" != xno], -- [AC_DEFINE(USE_HEALTH_CHECK_WRITE, 1, [Write when Checking Health])]) -+AS_IF([test "x$enable_health_write" != xno], [ -+ AC_DEFINE(USE_HEALTH_CHECK_WRITE, 1, [Write when Checking Health]) -+ AC_SUBST(ENABLE_HEALTH_WRITE, yes) -+], [ -+ AC_SUBST(ENABLE_HEALTH_WRITE, no) -+]) - ]) # LC_CONFIG_HEALTH_CHECK_WRITE - - # -@@ -199,8 +215,12 @@ AC_ARG_ENABLE([lru_resize], - [enable lru resize support]), - [], [enable_lru_resize="yes"]) - AC_MSG_RESULT([$enable_lru_resize]) --AS_IF([test "x$enable_lru_resize" != xno], -- [AC_DEFINE(HAVE_LRU_RESIZE_SUPPORT, 1, [Enable lru resize support])]) -+AS_IF([test "x$enable_lru_resize" != xno], [ -+ AC_DEFINE(HAVE_LRU_RESIZE_SUPPORT, 1, [Enable lru resize support]) -+ AC_SUBST(ENABLE_LRU_RESIZE, yes) -+], [ -+ AC_SUBST(ENABLE_LRU_RESIZE, no) -+]) - ]) # LC_CONFIG_LRU_RESIZE - - # -@@ -3311,8 +3331,12 @@ AC_ARG_ENABLE([mindf], - [Make statfs report the minimum available space on any single OST instead of the sum of free space on all OSTs]), - [], [enable_mindf="no"]) - AC_MSG_RESULT([$enable_mindf]) --AS_IF([test "$enable_mindf" = "yes"], -- [AC_DEFINE([MIN_DF], 1, [Report minimum OST free space])]) -+AS_IF([test "$enable_mindf" = "yes"], [ -+ AC_DEFINE([MIN_DF], 1, [Report minimum OST free space]) -+ AC_SUBST(ENABLE_MINDF, yes) -+], [ -+ AC_SUBST(ENABLE_MINDF, no) -+]) - - AC_MSG_CHECKING([whether to randomly failing memory alloc]) - AC_ARG_ENABLE([fail_alloc], -@@ -3320,9 +3344,12 @@ AC_ARG_ENABLE([fail_alloc], - [disable randomly alloc failure]), - [], [enable_fail_alloc="yes"]) - AC_MSG_RESULT([$enable_fail_alloc]) --AS_IF([test "x$enable_fail_alloc" != xno], -- [AC_DEFINE([RANDOM_FAIL_ALLOC], 1, -- [enable randomly alloc failure])]) -+AS_IF([test "x$enable_fail_alloc" != xno], [ -+ AC_DEFINE([RANDOM_FAIL_ALLOC], 1, [enable randomly alloc failure]) -+ AC_SUBST(ENABLE_FAIL_ALLOC, yes) -+], [ -+ AC_SUBST(ENABLE_FAIL_ALLOC, no) -+]) - - AC_MSG_CHECKING([whether to check invariants (expensive cpu-wise)]) - AC_ARG_ENABLE([invariants], -@@ -3330,9 +3357,13 @@ AC_ARG_ENABLE([invariants], - [enable invariant checking (cpu intensive)]), - [], [enable_invariants="no"]) - AC_MSG_RESULT([$enable_invariants]) --AS_IF([test "x$enable_invariants" = xyes], -- [AC_DEFINE([CONFIG_LUSTRE_DEBUG_EXPENSIVE_CHECK], 1, -- [enable invariant checking])]) -+AS_IF([test "x$enable_invariants" = xyes], [ -+ AC_DEFINE([CONFIG_LUSTRE_DEBUG_EXPENSIVE_CHECK], 1, -+ [enable invariant checking]) -+ AC_SUBST(ENABLE_INVARIANTS, yes) -+], [ -+ AC_SUBST(ENABLE_INVARIANTS, no) -+]) - - AC_MSG_CHECKING([whether to track references with lu_ref]) - AC_ARG_ENABLE([lu_ref], -@@ -3340,9 +3371,13 @@ AC_ARG_ENABLE([lu_ref], - [enable lu_ref reference tracking code]), - [], [enable_lu_ref="no"]) - AC_MSG_RESULT([$enable_lu_ref]) --AS_IF([test "x$enable_lu_ref" = xyes], -- [AC_DEFINE([CONFIG_LUSTRE_DEBUG_LU_REF], 1, -- [enable lu_ref reference tracking code])]) -+AS_IF([test "x$enable_lu_ref" = xyes], [ -+ AC_DEFINE([CONFIG_LUSTRE_DEBUG_LU_REF], 1, -+ [enable lu_ref reference tracking code]) -+ AC_SUBST(ENABLE_LU_REF, yes) -+], [ -+ AC_SUBST(ENABLE_LU_REF, no) -+]) - - AC_MSG_CHECKING([whether to enable page state tracking]) - AC_ARG_ENABLE([pgstate-track], -@@ -3350,9 +3385,13 @@ AC_ARG_ENABLE([pgstate-track], - [enable page state tracking]), - [], [enable_pgstat_track="no"]) - AC_MSG_RESULT([$enable_pgstat_track]) --AS_IF([test "x$enable_pgstat_track" = xyes], -- [AC_DEFINE([CONFIG_DEBUG_PAGESTATE_TRACKING], 1, -- [enable page state tracking code])]) -+AS_IF([test "x$enable_pgstat_track" = xyes], [ -+ AC_DEFINE([CONFIG_DEBUG_PAGESTATE_TRACKING], 1, -+ [enable page state tracking code]) -+ AC_SUBST(ENABLE_PGSTAT_TRACK, yes) -+], [ -+ AC_SUBST(ENABLE_PGSTAT_TRACK, no) -+]) - - PKG_PROG_PKG_CONFIG - AC_MSG_CHECKING([systemd unit file directory]) -diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c -index bf61555c33..1d7a29d76f 100644 ---- a/lustre/ldlm/ldlm_lib.c -+++ b/lustre/ldlm/ldlm_lib.c -@@ -465,7 +465,7 @@ int client_obd_setup(struct obd_device *obd, struct lustre_cfg *lcfg) - - cli->cl_supp_cksum_types = OBD_CKSUM_CRC32; - cli->cl_preferred_cksum_type = 0; --#ifdef ENABLE_CHECKSUM -+#ifdef CONFIG_ENABLE_CHECKSUM - /* Turn on checksumming by default. */ - cli->cl_checksum = 1; - /* -diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c -index dfb6b185b2..f67ea3f7eb 100644 ---- a/lustre/llite/llite_lib.c -+++ b/lustre/llite/llite_lib.c -@@ -162,10 +162,10 @@ static struct ll_sb_info *ll_init_sbi(void) - atomic_set(&sbi->ll_ra_info.ra_async_inflight, 0); - - set_bit(LL_SBI_VERBOSE, sbi->ll_flags); --#ifdef ENABLE_CHECKSUM -+#ifdef CONFIG_ENABLE_CHECKSUM - set_bit(LL_SBI_CHECKSUM, sbi->ll_flags); - #endif --#ifdef ENABLE_FLOCK -+#ifdef CONFIG_ENABLE_FLOCK - set_bit(LL_SBI_FLOCK, sbi->ll_flags); - #endif - --- -2.33.0 - diff --git a/0030-LU-15193-quota-expand-QUOTA_MAX_TRANSIDS-to-12.patch b/0030-LU-15193-quota-expand-QUOTA_MAX_TRANSIDS-to-12.patch deleted file mode 100644 index 8e3843fc1a341a5e9af9e7be47304a8e8bcc4ce3..0000000000000000000000000000000000000000 --- a/0030-LU-15193-quota-expand-QUOTA_MAX_TRANSIDS-to-12.patch +++ /dev/null @@ -1,89 +0,0 @@ -From c20d23cd92c5bc748a618e9ed96e6eddd794ab45 Mon Sep 17 00:00:00 2001 -From: Lei Feng -Date: Thu, 4 Nov 2021 19:41:06 +0800 -Subject: [PATCH 30/61] LU-15193 quota: expand QUOTA_MAX_TRANSIDS to 12 - -In some rare cases 12 quota ids are needed. -Usually (user, group) * (block, inode) * (inode, parent) = 8 qids -are needed. But with project id, -(user, group, project) * (block, inode) * (inode, parent) = 12 qids -are needed. - -Lustre-change: https://review.whamcloud.com/45456 -Lustre-commit: 61481796ac85e9ab2469b8d2f4cc75088c65d298 - -Change-Id: I4b3ee197f6e274abda06edf60b246f089fe28d10 -Signed-off-by: Lei Feng -Test-Parameters: trivial testlist=sanity-quota -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49611 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Stephane Thiell -Reviewed-by: Oleg Drokin -Reviewed-by: Andreas Dilger ---- - lustre/include/lustre_quota.h | 5 +++-- - lustre/tests/sanity-quota.sh | 31 +++++++++++++++++++++++++++++++ - 2 files changed, 34 insertions(+), 2 deletions(-) - -diff --git a/lustre/include/lustre_quota.h b/lustre/include/lustre_quota.h -index 4b674d8b12..6b98f4c9a2 100644 ---- a/lustre/include/lustre_quota.h -+++ b/lustre/include/lustre_quota.h -@@ -234,8 +234,9 @@ struct lquota_id_info { - * a single transaction for inode and block quota, which is chown transaction: - * original uid and gid, new uid and gid. - * -- * This value might need to be revised when directory quota is added. */ --#define QUOTA_MAX_TRANSIDS 8 -+ * Given a parent dir and a sub dir, with different uid, gid and project id, -+ * need x x = 12 ids */ -+#define QUOTA_MAX_TRANSIDS 12 - - /* all qids involved in a single transaction */ - struct lquota_trans { -diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh -index f36601ce5d..5b9ede2d26 100755 ---- a/lustre/tests/sanity-quota.sh -+++ b/lustre/tests/sanity-quota.sh -@@ -5414,6 +5414,37 @@ test_81() { - } - run_test 81 "Race qmt_start_pool_recalc with qmt_pool_free" - -+test_82() -+{ -+ (( $MDS1_VERSION >= $(version_code 2.14.55) )) || -+ skip "need MDS 2.14.55 or later" -+ is_project_quota_supported || -+ skip "skip project quota unsupported" -+ -+ setup_quota_test || error "setup quota failed with $?" -+ stack_trap cleanup_quota_test -+ quota_init -+ -+ local parent_dir="$DIR/$tdir.parent" -+ local child_dir="$parent_dir/child" -+ -+ mkdir -p $child_dir -+ stack_trap "chown -R 0:0 $parent_dir" -+ -+ chown $TSTUSR:$TSTUSR $parent_dir || -+ error "failed to chown on $parent_dir" -+ chown $TSTUSR2:$TSTUSRS2 $child_dir || -+ error "failed to chown on $parent_dir" -+ -+ $LFS project -p 1000 $parent_dir || -+ error "failed to set project id on $parent_dir" -+ $LFS project -p 1001 $child_dir || -+ error "failed to set project id on $child_dir" -+ -+ rmdir $child_dir || error "cannot remove child dir, test failed" -+} -+run_test 82 "verify more than 8 qids for single operation" -+ - quota_fini() - { - do_nodes $(comma_list $(nodes_list)) \ --- -2.33.0 - diff --git a/0040-LU-16824-ldiskfs-add-support-for-openEuler-22.03-LTS.patch b/0030-LU-16824-ldiskfs-add-support-for-openEuler-22.03-LTS.patch similarity index 98% rename from 0040-LU-16824-ldiskfs-add-support-for-openEuler-22.03-LTS.patch rename to 0030-LU-16824-ldiskfs-add-support-for-openEuler-22.03-LTS.patch index 2e51ba4e5d2990eaae5679505d8cb82b612534d4..88130e93e80d3ca2bd94894166d971467fe6f2b8 100644 --- a/0040-LU-16824-ldiskfs-add-support-for-openEuler-22.03-LTS.patch +++ b/0030-LU-16824-ldiskfs-add-support-for-openEuler-22.03-LTS.patch @@ -1,7 +1,7 @@ -From 1d3b7b31f3f0f4c9040e16ade9b333e32c94bbae Mon Sep 17 00:00:00 2001 +From 36c98b6e70946896792789c6b0f65ef35ba400d7 Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Wed, 10 May 2023 10:08:38 +0000 -Subject: [PATCH 40/61] LU-16824 ldiskfs: add support for openEuler 22.03 LTS +Subject: [PATCH 30/49] LU-16824 ldiskfs: add support for openEuler 22.03 LTS SP1 Add openEuler 22.03 LTS SP1 config target file. @@ -32,10 +32,10 @@ Signed-off-by: Xinliang Liu create mode 100644 lustre/kernel_patches/targets/5.10-oe2203sp1.target.in diff --git a/config/lustre-build-ldiskfs.m4 b/config/lustre-build-ldiskfs.m4 -index 8bc334b951..f0e987b010 100644 +index 9ece25f757..b521b29be7 100644 --- a/config/lustre-build-ldiskfs.m4 +++ b/config/lustre-build-ldiskfs.m4 -@@ -126,6 +126,7 @@ AS_IF([test x$RHEL_KERNEL = xyes], [ +@@ -127,6 +127,7 @@ AS_IF([test x$RHEL_KERNEL = xyes], [ ], [test x$OPENEULER_KERNEL = xyes], [ case $OPENEULER_VERSION_NO in 2203.0) LDISKFS_SERIES="5.10.0-oe2203.series" ;; @@ -44,7 +44,7 @@ index 8bc334b951..f0e987b010 100644 ]) ]) diff --git a/contrib/lbuild/funcs.sh b/contrib/lbuild/funcs.sh -index 78e2af9468..8db9532b67 100644 +index 25978095d4..2323b31fb8 100644 --- a/contrib/lbuild/funcs.sh +++ b/contrib/lbuild/funcs.sh @@ -156,8 +156,17 @@ autodetect_distro() { @@ -65,8 +65,8 @@ index 78e2af9468..8db9532b67 100644 ;; *) fatal 1 "I don't know what distro name $name and version $version is.\nEither update autodetect_distro() or use the --distro argument." -@@ -213,6 +222,7 @@ autodetect_target() { - sles15.4) target="$(uname -r | cut -d . -f 1,2)-sles15sp4";; +@@ -215,6 +224,7 @@ autodetect_target() { + sles15.5) target="$(uname -r | cut -d . -f 1,2)-sles15sp5";; fc18) target="3.x-fc18";; oe2203) target="5.10-oe2203";; + oe2203sp1) target="5.10-oe2203sp1";; @@ -963,10 +963,10 @@ index 0000000000..1cc20c8e58 +oe2203/ext4-delayed-iput.patch +rhel8/ext4-old_ea_inodes_handling_fix.patch diff --git a/lustre.spec.in b/lustre.spec.in -index a8c0934428..6ef7960300 100644 +index 1ddb1f9224..39f7bac487 100644 --- a/lustre.spec.in +++ b/lustre.spec.in -@@ -214,6 +214,9 @@ BuildRequires: pkgconfig +@@ -215,6 +215,9 @@ BuildRequires: pkgconfig %else %if "%{_vendor}" == "openEuler" BuildRequires: openEuler-rpm-config @@ -977,10 +977,10 @@ index a8c0934428..6ef7960300 100644 BuildRequires: pkg-config %endif diff --git a/lustre/ChangeLog b/lustre/ChangeLog -index 9401e95187..18c0649a11 100644 +index d58aa724bc..6a8dbf5f6c 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog -@@ -101,6 +101,7 @@ TBD Whamcloud +@@ -105,6 +105,7 @@ vanilla linux 5.4.21 (ZFS + ldiskfs) vanilla linux 5.4.136 (ZFS + ldiskfs) 5.10.0-60.94.0.118.oe2203 (openEuler 22.03 LTS) @@ -989,11 +989,11 @@ index 9401e95187..18c0649a11 100644 * Client primary kernels built and tested during release cycle: 5.14.0-284.11.1.el9 (RHEL9.2) diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 -index 12afe95fbc..2d4ad45d71 100644 +index a88ccc6251..3d7ed3ed2c 100644 --- a/lustre/autoconf/lustre-core.m4 +++ b/lustre/autoconf/lustre-core.m4 -@@ -3497,6 +3497,7 @@ lustre/kernel_patches/targets/5.3-sles15sp3.target - lustre/kernel_patches/targets/5.14-sles15sp4.target +@@ -4475,6 +4475,7 @@ lustre/kernel_patches/targets/5.14-sles15sp4.target + lustre/kernel_patches/targets/5.14-sles15sp5.target lustre/kernel_patches/targets/3.x-fc18.target lustre/kernel_patches/targets/5.10-oe2203.target +lustre/kernel_patches/targets/5.10-oe2203sp1.target diff --git a/0031-LU-16916-tests-fix-client_evicted-not-to-ignore-EOPN.patch b/0031-LU-16916-tests-fix-client_evicted-not-to-ignore-EOPN.patch deleted file mode 100644 index a8c67788c0e16f7b7473b3f8c5969e1b944b0851..0000000000000000000000000000000000000000 --- a/0031-LU-16916-tests-fix-client_evicted-not-to-ignore-EOPN.patch +++ /dev/null @@ -1,76 +0,0 @@ -From ff9e29d3b534959261950e45595f970d7f39213f Mon Sep 17 00:00:00 2001 -From: Jian Yu -Date: Fri, 14 Jul 2023 13:22:18 +0800 -Subject: [PATCH 31/61] LU-16916 tests: fix client_evicted() not to ignore - EOPNOTSUPP - -After RHEL 9.x or Ubuntu 22.04 client is evicted, "lfs df" returns -error code 95 (EOPNOTSUPP), which is ignored in check_lfs_df_ret_val() -and then causes client_evicted() to ingore that error. - -This patch fixes client_evicted() to check the return value -from "lfs df" directly so as not to ignore EOPNOTSUPP. - -Lustre-change: https://review.whamcloud.com/51667 -Lustre-commit: a5a9ded43b72238c2df8e0a74f03151ea3d4ce99 - -Test-Parameters: trivial clientdistro=el9.2 testlist=replay-vbr -Test-Parameters: trivial clientdistro=el8.8 testlist=replay-vbr - -Change-Id: I633ae8769fc563b8068f433e2afae29463ac5553 -Signed-off-by: Jian Yu -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51668 -Reviewed-by: Oleg Drokin -Reviewed-by: Andreas Dilger -Reviewed-by: Arshad Hussain -Tested-by: Maloo -Tested-by: jenkins ---- - lustre/tests/test-framework.sh | 16 +++++++++++++--- - 1 file changed, 13 insertions(+), 3 deletions(-) - -diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh -index 2c92993117..6126541ef4 100755 ---- a/lustre/tests/test-framework.sh -+++ b/lustre/tests/test-framework.sh -@@ -3583,16 +3583,25 @@ wait_remote_prog () { - return $rc - } - --lfs_df_check() { -+_lfs_df_check() { - local clients=${1:-$CLIENTS} - local rc=0 - -- if [ -z "$clients" ]; then -+ if [[ -z "$clients" ]]; then - $LFS df $MOUNT > /dev/null || rc=$? - else - $PDSH $clients "$LFS df $MOUNT" > /dev/null || rc=$? - fi - -+ return $rc -+} -+ -+lfs_df_check() { -+ local clients=${1:-$CLIENTS} -+ local rc=0 -+ -+ _lfs_df_check "$clients" || rc=$? -+ - check_lfs_df_ret_val $rc - } - -@@ -3622,7 +3631,8 @@ client_up() { - } - - client_evicted() { -- ! client_up $1 -+ sleep 1 -+ ! _lfs_df_check $1 - } - - client_reconnect_try() { --- -2.33.0 - diff --git a/0041-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP2.patch b/0031-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP2.patch similarity index 93% rename from 0041-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP2.patch rename to 0031-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP2.patch index 26a1bc1b2910293de8a1e9cc50399b186b8a83f4..57e692b6d1ba67843682284298de2468ee20d110 100644 --- a/0041-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP2.patch +++ b/0031-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP2.patch @@ -1,7 +1,7 @@ -From 38a03676580e5f2925d30fd1a2459cc9b0237d60 Mon Sep 17 00:00:00 2001 +From 66d0041fe7565ba9a028dece85507aa13b154d99 Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Tue, 18 Jul 2023 03:42:19 +0000 -Subject: [PATCH 41/61] LU-16976 ldiskfs: add support for openEuler 22.03 SP2 +Subject: [PATCH 31/49] LU-16976 ldiskfs: add support for openEuler 22.03 SP2 Add ldiskfs server support for oe2203sp2. Also refine openEuler lbuild scripts. @@ -26,20 +26,20 @@ Signed-off-by: Xinliang Liu create mode 100644 lustre/kernel_patches/targets/5.10-oe2203sp2.target.in diff --git a/config/lustre-build-ldiskfs.m4 b/config/lustre-build-ldiskfs.m4 -index f0e987b010..c225a6abaa 100644 +index b521b29be7..c520febcd9 100644 --- a/config/lustre-build-ldiskfs.m4 +++ b/config/lustre-build-ldiskfs.m4 -@@ -126,7 +126,7 @@ AS_IF([test x$RHEL_KERNEL = xyes], [ +@@ -127,7 +127,7 @@ AS_IF([test x$RHEL_KERNEL = xyes], [ ], [test x$OPENEULER_KERNEL = xyes], [ case $OPENEULER_VERSION_NO in 2203.0) LDISKFS_SERIES="5.10.0-oe2203.series" ;; - 2203.1) LDISKFS_SERIES="5.10.0-oe2203sp1.series" ;; -+ 2203.1|2203.2) LDISKFS_SERIES="5.10.0-oe2203sp1.series" ;; ++ 2203.*) LDISKFS_SERIES="5.10.0-oe2203sp1.series" ;; esac ]) ]) diff --git a/contrib/lbuild/funcs.sh b/contrib/lbuild/funcs.sh -index 8db9532b67..78389a5d96 100644 +index 2323b31fb8..3261a4752f 100644 --- a/contrib/lbuild/funcs.sh +++ b/contrib/lbuild/funcs.sh @@ -165,7 +165,7 @@ autodetect_distro() { @@ -51,8 +51,8 @@ index 8db9532b67..78389a5d96 100644 fi ;; *) -@@ -222,7 +222,8 @@ autodetect_target() { - sles15.4) target="$(uname -r | cut -d . -f 1,2)-sles15sp4";; +@@ -224,7 +224,8 @@ autodetect_target() { + sles15.5) target="$(uname -r | cut -d . -f 1,2)-sles15sp5";; fc18) target="3.x-fc18";; oe2203) target="5.10-oe2203";; - oe2203sp1) target="5.10-oe2203sp1";; @@ -227,10 +227,10 @@ index 0000000000..35845f336b +EOF +)" diff --git a/lustre/ChangeLog b/lustre/ChangeLog -index 18c0649a11..3f03823e3d 100644 +index 6a8dbf5f6c..7506abaaf1 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog -@@ -102,6 +102,7 @@ TBD Whamcloud +@@ -106,6 +106,7 @@ vanilla linux 5.4.136 (ZFS + ldiskfs) 5.10.0-60.94.0.118.oe2203 (openEuler 22.03 LTS) 5.10.0-136.32.0.108.oe2203sp1 (openEuler 22.03 LTS SP1) @@ -239,10 +239,10 @@ index 18c0649a11..3f03823e3d 100644 * Client primary kernels built and tested during release cycle: 5.14.0-284.11.1.el9 (RHEL9.2) diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 -index 2d4ad45d71..feed23e01f 100644 +index 3d7ed3ed2c..4657ebe337 100644 --- a/lustre/autoconf/lustre-core.m4 +++ b/lustre/autoconf/lustre-core.m4 -@@ -3498,6 +3498,7 @@ lustre/kernel_patches/targets/5.14-sles15sp4.target +@@ -4476,6 +4476,7 @@ lustre/kernel_patches/targets/5.14-sles15sp5.target lustre/kernel_patches/targets/3.x-fc18.target lustre/kernel_patches/targets/5.10-oe2203.target lustre/kernel_patches/targets/5.10-oe2203sp1.target diff --git a/0032-LU-16626-build-remove-python2-dependencies.patch b/0032-LU-16626-build-remove-python2-dependencies.patch deleted file mode 100644 index 5c1451ad878660d49a0b1f734482c1dd7dc0b611..0000000000000000000000000000000000000000 --- a/0032-LU-16626-build-remove-python2-dependencies.patch +++ /dev/null @@ -1,108 +0,0 @@ -From 53f40033ccf9eeb4155b18f4046f33b19252aa18 Mon Sep 17 00:00:00 2001 -From: Alex Deiter -Date: Thu, 9 Mar 2023 18:09:19 +0400 -Subject: [PATCH 32/61] LU-16626 build: remove python2 dependencies - -Fixed packaging issue caused by zfsobj2fid script. - -Lustre-change: https://review.whamcloud.com/50241 -Lustre-commit: 404a1e827b0a9d86864695c8699e1ca076be6c9d - -Test-Parameters: trivial -Signed-off-by: Alex Deiter -Change-Id: I4375038b0d2c2b42ac4080fe834d35bdd3ef54f8 -Reviewed-by: Minh Diep -Reviewed-by: Jian Yu -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51426 -Reviewed-by: Oleg Drokin -Tested-by: jenkins -Tested-by: Maloo ---- - lustre/scripts/zfsobj2fid | 47 ++++++++++++++++++++++----------------- - 1 file changed, 26 insertions(+), 21 deletions(-) - -diff --git a/lustre/scripts/zfsobj2fid b/lustre/scripts/zfsobj2fid -index f7ae96bd17..a76095e1de 100755 ---- a/lustre/scripts/zfsobj2fid -+++ b/lustre/scripts/zfsobj2fid -@@ -1,4 +1,4 @@ --#!/usr/bin/env python2 -+#!/usr/bin/env python3 - - # Copyright (c) 2014, Lawrence Livermore National Security, LLC. - # Produced at the Lawrence Livermore National Laboratory. -@@ -27,21 +27,31 @@ - import sys - import subprocess - --def from_bytes(b): -- return sum(b[i] << i*8 for i in range(len(b))) -+def from_bytes(data): -+ return hex(sum(data[i] << i * 8 for i in range(len(data)))) - - def main(): - if len(sys.argv) != 3: -- print "Usage:", sys.argv[0], " " -+ print('Usage:', sys.argv[0], '', '') - return 1 - -- p = subprocess.Popen(["zdb", "-e", "-vvv", sys.argv[1], sys.argv[2]], -- stdout=subprocess.PIPE) -- pout, perr = p.communicate() -+ cmd = ['zdb', '-e', '-vvv', sys.argv[1], sys.argv[2]] -+ process = subprocess.Popen(cmd, -+ stdout=subprocess.PIPE, -+ stderr=subprocess.PIPE, -+ universal_newlines=True) -+ stdout, stderr = process.communicate() -+ result = process.returncode - -- b = bytearray() -+ if result != 0: -+ msg = 'Error %d on %s: %s %s' % (result, cmd, stdout, stderr) -+ raise RuntimeError(msg) -+ -+ lines = stdout.splitlines() -+ data = bytearray() - found_fid = False -- for line in pout.split('\n'): -+ -+ for line in lines: - part = line.split() - if not part or part[0] != 'trusted.fid': - continue -@@ -53,24 +63,19 @@ def main(): - if val == '\\': - val = fid[0:3] - fid = fid[3:] -- b.append(int(val, 8)) -+ data.append(int(val, 8)) - else: -- b.append(ord(val)) -+ data.append(ord(val)) - break - - if not found_fid: -- print "FID not found on", sys.argv[1], sys.argv[2] -+ print('FID not found on', sys.argv[1], sys.argv[2]) - return 1 - -- print '[' \ -- + hex(from_bytes(b[0:8])) \ -- + ':' \ -- + hex(from_bytes(b[8:12])) \ -- + ':' \ -- + hex(from_bytes(b[12:16])) \ -- + ']' -- -+ print('[%s:%s:%s]' % (from_bytes(data[0:8]), -+ from_bytes(data[8:12]), -+ from_bytes(data[12:16]))) - return 0 - - if __name__ == '__main__': -- sys.exit(main()) -+ sys.exit(main()) --- -2.33.0 - diff --git a/0042-lustre.spec.in-match-rpm-macro-openEuler-for-openEul.patch b/0032-lustre.spec.in-match-rpm-macro-openEuler-for-openEul.patch similarity index 83% rename from 0042-lustre.spec.in-match-rpm-macro-openEuler-for-openEul.patch rename to 0032-lustre.spec.in-match-rpm-macro-openEuler-for-openEul.patch index 0b8d21c49bb91236186d2b9fc399a342aa883fe2..c489d570073539153b589261831b69ad5b3c63ee 100644 --- a/0042-lustre.spec.in-match-rpm-macro-openEuler-for-openEul.patch +++ b/0032-lustre.spec.in-match-rpm-macro-openEuler-for-openEul.patch @@ -1,7 +1,7 @@ -From bf5b9a174b2ec1d1a26c09e7c5d4b928a24aaa8c Mon Sep 17 00:00:00 2001 +From 0e4a99c6c7de75e0061e223ccb47f21c21bf8323 Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Mon, 7 Aug 2023 10:18:49 +0000 -Subject: [PATCH 42/61] lustre.spec.in: match rpm macro openEuler for openEuler +Subject: [PATCH 32/49] lustre.spec.in: match rpm macro openEuler for openEuler Linux So that it can handle openEuler derived OSes, like KylinOS. @@ -12,10 +12,10 @@ Signed-off-by: Xinliang Liu 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lustre.spec.in b/lustre.spec.in -index 6ef7960300..9aa271f504 100644 +index 39f7bac487..82fec51ab2 100644 --- a/lustre.spec.in +++ b/lustre.spec.in -@@ -136,7 +136,7 @@ +@@ -137,7 +137,7 @@ # requires want to set a version including epoch %global krequires %(echo %{kver} | sed -e 's/\.x86_64$//' -e 's/\.i[3456]86$//' -e 's/-smp$//' -e 's/-bigsmp$//' -e 's/[-.]ppc64$//' -e 's/\.aarch64$//' -e 's/-default$//' -e 's/-%{_flavor}//') @@ -24,7 +24,7 @@ index 6ef7960300..9aa271f504 100644 %global requires_kmod_name kmod-%{lustre_name} %global requires_kmod_osd_zfs_name kmod-%{lustre_name}-osd-zfs %if %{with lustre_tests} -@@ -176,7 +176,7 @@ +@@ -177,7 +177,7 @@ %endif # openEuler comes with systemd @@ -33,7 +33,7 @@ index 6ef7960300..9aa271f504 100644 %define with_systemd 1 %endif -@@ -212,7 +212,7 @@ BuildRequires: libtool libyaml-devel zlib-devel libnl3-devel flex bison +@@ -213,7 +213,7 @@ BuildRequires: libtool libyaml-devel zlib-devel libnl3-devel flex bison BuildRequires: redhat-rpm-config BuildRequires: pkgconfig %else @@ -42,7 +42,7 @@ index 6ef7960300..9aa271f504 100644 BuildRequires: openEuler-rpm-config %if %{with ldiskfs} BuildRequires: kernel-debugsource -@@ -231,20 +231,20 @@ Provides: lustre-server = %{version}-%{release} +@@ -232,7 +232,7 @@ Provides: lustre-server = %{version}-%{release} %endif Obsoletes: lustre-client < %{version} Provides: lustre-client = %{version}-%{release} @@ -51,6 +51,7 @@ index 6ef7960300..9aa271f504 100644 #suse don't support selinux BuildRequires: libselinux-devel %endif +@@ -242,13 +242,13 @@ BuildRequires: pkgconfig(libselinux) openssl-devel %if %{with lustre_modules} %if %{with mofed} BuildRequires: mlnx-ofa_kernel-devel @@ -66,7 +67,7 @@ index 6ef7960300..9aa271f504 100644 BuildRequires: kernel-rpm-macros %endif BuildRequires: %kernel_module_package_buildreqs -@@ -369,7 +369,7 @@ Provides: lustre-tests = %{version} +@@ -373,7 +373,7 @@ Provides: lustre-tests = %{version} Requires: lustre-iokit %endif Requires: lustre-devel = %{version} @@ -75,7 +76,7 @@ index 6ef7960300..9aa271f504 100644 Requires: python3 >= 3.6.0, python3-PyYAML %endif %if %{with lustre_modules} -@@ -385,7 +385,7 @@ Requires: attr, rsync, perl, lsof, /usr/bin/getconf +@@ -389,7 +389,7 @@ Requires: attr, rsync, perl, lsof, /usr/bin/getconf BuildRequires: mpich-devel %endif %if "%{mpi_name}" == "openmpi" @@ -84,7 +85,7 @@ index 6ef7960300..9aa271f504 100644 BuildRequires: openmpi-devel %else BuildRequires: openmpi2-devel -@@ -637,7 +637,7 @@ echo '%{_sysconfdir}/ha.d/resource.d/Lustre' >>lustre.files +@@ -642,7 +642,7 @@ echo '%{_sysconfdir}/ha.d/resource.d/Lustre' >>lustre.files echo '%{_unitdir}/lnet.service' >>lustre.files %endif diff --git a/0043-LU-15722-osd-ldiskfs-fix-IO-write-gets-stuck-for-64K.patch b/0033-LU-15722-osd-ldiskfs-fix-IO-write-gets-stuck-for-64K.patch similarity index 94% rename from 0043-LU-15722-osd-ldiskfs-fix-IO-write-gets-stuck-for-64K.patch rename to 0033-LU-15722-osd-ldiskfs-fix-IO-write-gets-stuck-for-64K.patch index a86193740904d83d1284ac9de0c2a949b8f81b01..41aa752cd201cd112cd0079c465e3f05818f8741 100644 --- a/0043-LU-15722-osd-ldiskfs-fix-IO-write-gets-stuck-for-64K.patch +++ b/0033-LU-15722-osd-ldiskfs-fix-IO-write-gets-stuck-for-64K.patch @@ -1,7 +1,7 @@ -From deacd917d127dda1ec2f22d8f4451205b03860ae Mon Sep 17 00:00:00 2001 +From 6701db5748f6c4c4ff768aa2bb18b078b87d8ec8 Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Wed, 6 Apr 2022 08:06:33 +0000 -Subject: [PATCH 43/61] LU-15722 osd-ldiskfs: fix IO write gets stuck for 64K +Subject: [PATCH 33/49] LU-15722 osd-ldiskfs: fix IO write gets stuck for 64K PAGE_SIZE This fixes below IO write stuck issue: @@ -45,7 +45,7 @@ Reviewed-by: Oleg Drokin 1 file changed, 44 insertions(+) diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c -index 7131add58d..dcc1081e0a 100644 +index f74862cd05..6df532bc00 100644 --- a/lustre/osd-ldiskfs/osd_io.c +++ b/lustre/osd-ldiskfs/osd_io.c @@ -1091,6 +1091,7 @@ static int osd_ldiskfs_map_inode_pages(struct inode *inode, @@ -56,7 +56,7 @@ index 7131add58d..dcc1081e0a 100644 int rc = 0, i = 0, mapped_index = 0; struct page *fp = NULL; int clen = 0; -@@ -1147,6 +1148,48 @@ static int osd_ldiskfs_map_inode_pages(struct inode *inode, +@@ -1148,6 +1149,48 @@ static int osd_ldiskfs_map_inode_pages(struct inode *inode, /* process found extent */ map.m_lblk = fp->index * blocks_per_page; map.m_len = blen = clen * blocks_per_page; @@ -105,7 +105,7 @@ index 7131add58d..dcc1081e0a 100644 cont_map: /** * We might restart transaction for block allocations, -@@ -1211,6 +1254,7 @@ cont_map: +@@ -1222,6 +1265,7 @@ cont_map: rc = 0; } diff --git a/0033-LU-16943-tests-use-primary-ost1-server-in-replay-sin.patch b/0033-LU-16943-tests-use-primary-ost1-server-in-replay-sin.patch deleted file mode 100644 index e55124be8d0be22b443f2daa40e45ee341a94962..0000000000000000000000000000000000000000 --- a/0033-LU-16943-tests-use-primary-ost1-server-in-replay-sin.patch +++ /dev/null @@ -1,48 +0,0 @@ -From db9108300af2949a7ff334973f3df4486cec4a6a Mon Sep 17 00:00:00 2001 -From: Jian Yu -Date: Wed, 23 Aug 2023 18:01:20 -0700 -Subject: [PATCH 33/61] LU-16943 tests: use primary ost1 server in - replay-single/135 - -This patch fixes replay-single test_135() to make sure -the primary ost1 server is used at the beginning of the test. - -Lustre-change: https://review.whamcloud.com/52058 -Lustre-commit: cdd8b056bff0d48155eaf4b7732d1d8880ceda55 - -Test-Parameters: trivial testlist=replay-single - -Test-Parameters: trivial env=FAILURE_MODE=HARD \ - clientcount=4 mdtcount=1 mdscount=2 osscount=2 \ - austeroptions=-R failover=true iscsi=1 \ - testlist=replay-single,mmp - -Fixes: 81418be83ed8 ("LU-16943 tests: fix replay-single/135 under hard failure mode") -Change-Id: Ia25314255c9f00ba71687e1f757517f37031caed -Signed-off-by: Jian Yu -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/52059 -Reviewed-by: Oleg Drokin -Reviewed-by: Alex Deiter -Tested-by: jenkins -Tested-by: Maloo ---- - lustre/tests/replay-single.sh | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh -index f631b73125..c02bcc2c18 100755 ---- a/lustre/tests/replay-single.sh -+++ b/lustre/tests/replay-single.sh -@@ -4958,6 +4958,9 @@ run_test 134 "replay creation of a file created in a pool" - - # LU-14027 - test_135() { -+ # make sure we are using the primary server -+ [[ $(facet_active ost1) == "ost1" ]] || fail ost1 -+ - mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed" - - # All files to ost1 --- -2.33.0 - diff --git a/0044-LU-15722-osd-ldiskfs-fix-write-stuck-for-64K-PAGE_SI.patch b/0034-LU-15722-osd-ldiskfs-fix-write-stuck-for-64K-PAGE_SI.patch similarity index 95% rename from 0044-LU-15722-osd-ldiskfs-fix-write-stuck-for-64K-PAGE_SI.patch rename to 0034-LU-15722-osd-ldiskfs-fix-write-stuck-for-64K-PAGE_SI.patch index 33a5ba1b9a891e5cf454c12433d2c6e3facd6ccb..eafdb00e91b69af12051cfd3e656d890b8cc8df3 100644 --- a/0044-LU-15722-osd-ldiskfs-fix-write-stuck-for-64K-PAGE_SI.patch +++ b/0034-LU-15722-osd-ldiskfs-fix-write-stuck-for-64K-PAGE_SI.patch @@ -1,7 +1,7 @@ -From 7b3bbf6fb4dace0af738bc69500558180a8e7227 Mon Sep 17 00:00:00 2001 +From d25542d4dc2ac339db9e2a23a68fb60167818580 Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Mon, 6 Jun 2022 08:59:54 +0000 -Subject: [PATCH 44/61] LU-15722 osd-ldiskfs: fix write stuck for 64K PAGE_SIZE +Subject: [PATCH 34/49] LU-15722 osd-ldiskfs: fix write stuck for 64K PAGE_SIZE This reverts a previous commit for large PAGE_SIZE to fix a stuck IO issue in another way. @@ -28,7 +28,7 @@ Signed-off-by: Xinliang Liu 2 files changed, 31 insertions(+), 54 deletions(-) diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h -index f06dde1b00..7300278b80 100644 +index f7b164ed94..ffedb94206 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -594,6 +594,8 @@ struct osd_iobuf { @@ -41,7 +41,7 @@ index f06dde1b00..7300278b80 100644 #define osd_dirty_inode(inode, flag) (inode)->i_sb->s_op->dirty_inode((inode), flag) diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c -index dcc1081e0a..233d88397c 100644 +index 6df532bc00..1af570e932 100644 --- a/lustre/osd-ldiskfs/osd_io.c +++ b/lustre/osd-ldiskfs/osd_io.c @@ -90,6 +90,14 @@ static int __osd_init_iobuf(struct osd_device *d, struct osd_iobuf *iobuf, @@ -97,7 +97,7 @@ index dcc1081e0a..233d88397c 100644 int rc = 0, i = 0, mapped_index = 0; struct page *fp = NULL; int clen = 0; -@@ -1150,44 +1149,19 @@ static int osd_ldiskfs_map_inode_pages(struct inode *inode, +@@ -1151,44 +1150,19 @@ static int osd_ldiskfs_map_inode_pages(struct inode *inode, map.m_len = blen = clen * blocks_per_page; /* @@ -153,7 +153,7 @@ index dcc1081e0a..233d88397c 100644 } cont_map: -@@ -1215,6 +1189,8 @@ cont_map: +@@ -1216,6 +1190,8 @@ cont_map: if (rc) GOTO(cleanup, rc); thandle->th_restart_tran = 1; @@ -162,7 +162,7 @@ index dcc1081e0a..233d88397c 100644 GOTO(cleanup, rc = -EAGAIN); } -@@ -1254,10 +1230,9 @@ cont_map: +@@ -1265,10 +1241,9 @@ cont_map: rc = 0; } diff --git a/0034-LU-16585-build-remove-python2-dependencies.patch b/0034-LU-16585-build-remove-python2-dependencies.patch deleted file mode 100644 index 48f695d4e5e2dc89eab280c0a0f572acd8180e3f..0000000000000000000000000000000000000000 --- a/0034-LU-16585-build-remove-python2-dependencies.patch +++ /dev/null @@ -1,164 +0,0 @@ -From cfc219ced11adf837e3bd1e356217f33cbe68317 Mon Sep 17 00:00:00 2001 -From: Alex Deiter -Date: Wed, 22 Feb 2023 02:27:47 +0400 -Subject: [PATCH 34/61] LU-16585 build: remove python2 dependencies - -Fixed packaging issue casued by scripts and control files. - -Lustre-change: https://review.whamcloud.com/50084 -Lustre-commit: bea3f81f84fd16d2d403682ef25b8abe314acd0f - -Test-Parameters: trivial -Signed-off-by: Alex Deiter -Change-Id: I6c9b24bf811269928494af17c15627902e5fe27b -Reviewed-by: Patrick Farrell -Reviewed-by: Feng Lei -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/52176 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - contrib/scripts/gerrit_checkpatch.py | 29 ++++++++++++++-------------- - debian/control | 2 +- - debian/control.main | 2 +- - 3 files changed, 17 insertions(+), 16 deletions(-) - -diff --git a/contrib/scripts/gerrit_checkpatch.py b/contrib/scripts/gerrit_checkpatch.py -index 9dbf1383ca..1563413abb 100755 ---- a/contrib/scripts/gerrit_checkpatch.py -+++ b/contrib/scripts/gerrit_checkpatch.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python2 -+#!/usr/bin/env python3 - # - # GPL HEADER START - # -@@ -33,6 +33,7 @@ Gerrit Checkpatch Reviewer Daemon - * POST reviews back to gerrit based on checkpatch output. - """ - -+from __future__ import print_function - import base64 - import fnmatch - import logging -@@ -41,7 +42,7 @@ import os - import requests - import subprocess - import time --import urllib -+from six.moves.urllib.parse import quote - - def _getenv_list(key, default=None, sep=':'): - """ -@@ -144,7 +145,7 @@ def parse_checkpatch_output(out, path_line_comments, warning_count): - except ValueError: - level, kind, message = None, None, None - -- if level != 'ERROR' and level != 'WARNING': -+ if level not in ('ERROR', 'WARNING'): - level, kind, message = None, None, None - - -@@ -155,9 +156,9 @@ def review_input_and_score(path_line_comments, warning_count): - """ - review_comments = {} - -- for path, line_comments in path_line_comments.iteritems(): -+ for path, line_comments in list(path_line_comments.items()): - path_comments = [] -- for line, comment_list in line_comments.iteritems(): -+ for line, comment_list in list(line_comments.items()): - message = '\n'.join(comment_list) - path_comments.append({'line': line, 'message': message}) - review_comments[path] = path_comments -@@ -194,7 +195,7 @@ def review_input_and_score(path_line_comments, warning_count): - - def _now(): - """_""" -- return long(time.time()) -+ return int(time.time()) - - - class Reviewer(object): -@@ -214,7 +215,7 @@ class Reviewer(object): - self.history_path = history_path - self.history_mode = 'rw' - self.history = {} -- self.timestamp = 0L -+ self.timestamp = 0 - self.post_enabled = True - self.post_interval = 10 - self.update_interval = 300 -@@ -292,7 +293,7 @@ class Reviewer(object): - for line in history_file: - epoch, change_id, revision, score = line.split() - if change_id == '-': -- self.timestamp = long(float(epoch)) -+ self.timestamp = int(float(epoch)) - else: - self.history[change_id + ' ' + revision] = score - -@@ -311,7 +312,7 @@ class Reviewer(object): - - if 'w' in self.history_mode: - with open(self.history_path, 'a') as history_file: -- print >> history_file, epoch, change_id, revision, score -+ print(epoch, change_id, revision, score, file=history_file) - - def in_history(self, change_id, revision): - """ -@@ -323,8 +324,8 @@ class Reviewer(object): - """ - GET one change by id. - """ -- path = ('/changes/' + urllib.quote(self.project, safe='') + '~' + -- urllib.quote(self.branch, safe='') + '~' + change_id + -+ path = ('/changes/' + quote(self.project, safe='') + '~' + -+ quote(self.branch, safe='') + '~' + change_id + - '?o=CURRENT_REVISION') - res = self._get(path) - if not res: -@@ -343,11 +344,11 @@ class Reviewer(object): - """ - query = dict(query) - project = query.get('project', self.project) -- query['project'] = urllib.quote(project, safe='') -+ query['project'] = quote(project, safe='') - branch = query.get('branch', self.branch) -- query['branch'] = urllib.quote(branch, safe='') -+ query['branch'] = quote(branch, safe='') - path = ('/changes/?q=' + -- '+'.join(k + ':' + v for k, v in query.iteritems()) + -+ '+'.join(k + ':' + v for k, v in list(query.items())) + - '&o=CURRENT_REVISION') - res = self._get(path) - if not res: -diff --git a/debian/control b/debian/control -index ac91ac2f24..836623edc4 100644 ---- a/debian/control -+++ b/debian/control -@@ -72,7 +72,7 @@ Package: lustre-iokit - Section: utils - Architecture: i386 armhf powerpc ppc64el amd64 ia64 arm64 - Priority: optional --Depends: lustre-client-utils (= ${binary:Version}), python2, perl, sg3-utils -+Depends: lustre-client-utils (= ${binary:Version}), python3, perl, sg3-utils - Description: Collection of benchmark tools for the Lustre filesystem - Lustre is a scalable, secure, robust, highly-available cluster file system. - This release is maintained by Whamcloud and available from -diff --git a/debian/control.main b/debian/control.main -index ac91ac2f24..836623edc4 100644 ---- a/debian/control.main -+++ b/debian/control.main -@@ -72,7 +72,7 @@ Package: lustre-iokit - Section: utils - Architecture: i386 armhf powerpc ppc64el amd64 ia64 arm64 - Priority: optional --Depends: lustre-client-utils (= ${binary:Version}), python2, perl, sg3-utils -+Depends: lustre-client-utils (= ${binary:Version}), python3, perl, sg3-utils - Description: Collection of benchmark tools for the Lustre filesystem - Lustre is a scalable, secure, robust, highly-available cluster file system. - This release is maintained by Whamcloud and available from --- -2.33.0 - diff --git a/0035-LU-15660-statahead-statahead-thread-doesn-t-stop.patch b/0035-LU-15660-statahead-statahead-thread-doesn-t-stop.patch deleted file mode 100644 index c58014eff9fa71dc1fc19912a7eb318aa92a4052..0000000000000000000000000000000000000000 --- a/0035-LU-15660-statahead-statahead-thread-doesn-t-stop.patch +++ /dev/null @@ -1,96 +0,0 @@ -From f7438d13b2aadbdf8e90e5b0b7732eeb20f3d475 Mon Sep 17 00:00:00 2001 -From: Yang Sheng -Date: Fri, 17 Jun 2022 20:30:34 +0800 -Subject: [PATCH 35/61] LU-15660 statahead: statahead thread doesn't stop - -Add a barrier to ensure sai_task changing can be seen -when access it without locking. Else the statahead -thread could sleep forever since wake_up was lost. - -Lustre-change: https://review.whamcloud.com/47673 -Lustre-commit: b977caa2dc7dddcec9e20d393ee79dfa9fe31c0d - -Signed-off-by: Yang Sheng -Change-Id: I211e99f1bdddaaaf028a205658f603fda034d389 -Reviewed-by: Neil Brown -Reviewed-by: Andreas Dilger -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/52300 -Tested-by: jenkins -Tested-by: Maloo -Reviewed-by: Oleg Drokin ---- - lustre/llite/statahead.c | 21 +++++++++++++-------- - 1 file changed, 13 insertions(+), 8 deletions(-) - -diff --git a/lustre/llite/statahead.c b/lustre/llite/statahead.c -index faf2860c8b..9df5137d9a 100644 ---- a/lustre/llite/statahead.c -+++ b/lustre/llite/statahead.c -@@ -1055,7 +1055,8 @@ static int ll_statahead_thread(void *arg) - if (!op_data) - GOTO(out, rc = -ENOMEM); - -- while (pos != MDS_DIR_END_OFF && sai->sai_task) { -+ /* matches smp_store_release() in ll_deauthorize_statahead() */ -+ while (pos != MDS_DIR_END_OFF && smp_load_acquire(&sai->sai_task)) { - struct lu_dirpage *dp; - struct lu_dirent *ent; - -@@ -1081,7 +1082,8 @@ static int ll_statahead_thread(void *arg) - - dp = page_address(page); - for (ent = lu_dirent_start(dp); -- ent != NULL && sai->sai_task && -+ /* matches smp_store_release() in ll_deauthorize_statahead() */ -+ ent != NULL && smp_load_acquire(&sai->sai_task) && - !sa_low_hit(sai); - ent = lu_dirent_next(ent)) { - __u64 hash; -@@ -1134,7 +1136,9 @@ static int ll_statahead_thread(void *arg) - fid_le_to_cpu(&fid, &ent->lde_fid); - - while (({set_current_state(TASK_IDLE); -- sai->sai_task; })) { -+ /* matches smp_store_release() in -+ * ll_deauthorize_statahead() */ -+ smp_load_acquire(&sai->sai_task); })) { - if (sa_has_callback(sai)) { - __set_current_state(TASK_RUNNING); - sa_handle_callback(sai); -@@ -1217,7 +1221,8 @@ static int ll_statahead_thread(void *arg) - * for file release to stop me. - */ - while (({set_current_state(TASK_IDLE); -- sai->sai_task; })) { -+ /* matches smp_store_release() in ll_deauthorize_statahead() */ -+ smp_load_acquire(&sai->sai_task); })) { - if (sa_has_callback(sai)) { - __set_current_state(TASK_RUNNING); - sa_handle_callback(sai); -@@ -1304,7 +1309,8 @@ void ll_deauthorize_statahead(struct inode *dir, void *key) - */ - struct task_struct *task = sai->sai_task; - -- sai->sai_task = NULL; -+ /* matches smp_load_acquire() in ll_statahead_thread() */ -+ smp_store_release(&sai->sai_task, NULL); - wake_up_process(task); - } - spin_unlock(&lli->lli_sa_lock); -@@ -1686,11 +1692,10 @@ static int start_statahead_thread(struct inode *dir, struct dentry *dentry, - GOTO(out, rc); - } - -- if (test_bit(LL_SBI_AGL_ENABLED, ll_i2sbi(parent->d_inode)->ll_flags) && -- agl) -+ if (test_bit(LL_SBI_AGL_ENABLED, sbi->ll_flags) && agl) - ll_start_agl(parent, sai); - -- atomic_inc(&ll_i2sbi(parent->d_inode)->ll_sa_total); -+ atomic_inc(&sbi->ll_sa_total); - sai->sai_task = task; - - wake_up_process(task); --- -2.33.0 - diff --git a/0045-LU-15978-osp-fix-striped-directory-deletion-fails-fo.patch b/0035-LU-15978-osp-fix-striped-directory-deletion-fails-fo.patch similarity index 98% rename from 0045-LU-15978-osp-fix-striped-directory-deletion-fails-fo.patch rename to 0035-LU-15978-osp-fix-striped-directory-deletion-fails-fo.patch index eec8f3a7972faf3d18effba28be949b6067227c7..38ee922f2614a255db25aff471199ae2c9aa027d 100644 --- a/0045-LU-15978-osp-fix-striped-directory-deletion-fails-fo.patch +++ b/0035-LU-15978-osp-fix-striped-directory-deletion-fails-fo.patch @@ -1,7 +1,7 @@ -From 3ef6abe3067ba1cb61d655d33ef129cf004a7ebc Mon Sep 17 00:00:00 2001 +From a052cac469037116d7e07964ad3793866610350c Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Tue, 28 Jun 2022 08:34:46 +0000 -Subject: [PATCH 45/61] LU-15978 osp: fix striped directory deletion fails for +Subject: [PATCH 35/49] LU-15978 osp: fix striped directory deletion fails for 64K PAGE_SIZE This fixes the rmdir errors below: diff --git a/0036-LU-16042-tests-can-not-get-cache-size-on-Arm64.patch b/0036-LU-16042-tests-can-not-get-cache-size-on-Arm64.patch deleted file mode 100644 index 230feeca894d87d47312951b5460100dc12b2a16..0000000000000000000000000000000000000000 --- a/0036-LU-16042-tests-can-not-get-cache-size-on-Arm64.patch +++ /dev/null @@ -1,51 +0,0 @@ -From a9e47f3bf9255047890d9aa886954432fe058ef5 Mon Sep 17 00:00:00 2001 -From: Kevin Zhao -Date: Mon, 25 Jul 2022 15:53:44 +0800 -Subject: [PATCH 36/61] LU-16042 tests: can not get cache size on Arm64 - -This fix the test fail on Arm64, the cache size can not be -display on /proc/cpuinfo. And even in the VM and somee -older Arm64 CPU, we can not get the cachesize. So it's -better to fallback to a pre-set value here if we don't get -the cache size. - -Lustre-change: https://review.whamcloud.com/48030 -Lustre-commit: f276f1cb0859e8718448e69bd99ee305f5e62d42 - -Test-Parameters: trivial -Test-Parameters: clientarch=aarch64 clientdistro=el8.7 \ - testlist=sanity env=ONLY=155 - -Signed-off-by: Kevin Zhao -Change-Id: I17ce1d8accc69d1489db2071a2741b3927fff302 -Reviewed-by: Andreas Dilger -Reviewed-by: James Simmons -Signed-off-by: Xinliang Liu -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51179 -Tested-by: Maloo -Tested-by: jenkins -Reviewed-by: Oleg Drokin ---- - lustre/tests/sanity.sh | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh -index 93f8c3b30c..7b4d43ef12 100755 ---- a/lustre/tests/sanity.sh -+++ b/lustre/tests/sanity.sh -@@ -15602,6 +15602,12 @@ test_155_big_load() { - free_min_max - local cache_size=$(do_facet ost$((MAXI+1)) \ - "awk '/cache/ {sum+=\\\$4} END {print sum}' /proc/cpuinfo") -+ -+ # LU-16042: can not get the cache size on Arm64 VM here, fallback to a -+ # pre-set value -+ if [ -z "$cache_size" ]; then -+ cache_size=256 -+ fi - local large_file_size=$((cache_size * 2)) - - echo "OSS cache size: $cache_size KB" --- -2.33.0 - diff --git a/0046-ldiskfs-add-support-for-oe2003.patch b/0036-ldiskfs-add-support-for-oe2003.patch similarity index 99% rename from 0046-ldiskfs-add-support-for-oe2003.patch rename to 0036-ldiskfs-add-support-for-oe2003.patch index 70416f1ff726cd168f8736224a76869068b572a9..3c55d9372bf633013e5c385efd0f747b8818497e 100644 --- a/0046-ldiskfs-add-support-for-oe2003.patch +++ b/0036-ldiskfs-add-support-for-oe2003.patch @@ -1,7 +1,7 @@ -From 7325c6cf0aeb80aeefe1cc864decc4ce02a9fe01 Mon Sep 17 00:00:00 2001 +From 40ad24b409dc4627698cf0cd37ccb5657d65cc40 Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Thu, 10 Aug 2023 09:34:02 +0000 -Subject: [PATCH 46/61] ldiskfs: add support for oe2003 +Subject: [PATCH 36/49] ldiskfs: add support for oe2003 Signed-off-by: Xinliang Liu --- @@ -32,12 +32,12 @@ Signed-off-by: Xinliang Liu create mode 100644 lustre/kernel_patches/targets/4.19-oe2003sp3.target.in diff --git a/config/lustre-build-ldiskfs.m4 b/config/lustre-build-ldiskfs.m4 -index c225a6abaa..51ddcd07e1 100644 +index c520febcd9..a7bc057687 100644 --- a/config/lustre-build-ldiskfs.m4 +++ b/config/lustre-build-ldiskfs.m4 -@@ -128,6 +128,12 @@ AS_IF([test x$RHEL_KERNEL = xyes], [ +@@ -129,6 +129,12 @@ AS_IF([test x$RHEL_KERNEL = xyes], [ 2203.0) LDISKFS_SERIES="5.10.0-oe2203.series" ;; - 2203.1|2203.2) LDISKFS_SERIES="5.10.0-oe2203sp1.series" ;; + 2203.*) LDISKFS_SERIES="5.10.0-oe2203sp1.series" ;; esac + + AS_IF([test -z "$OPENEULER_VERSION_NO"], [ @@ -83,12 +83,12 @@ index 1ff6c47856..6cdcba0b60 100644 AS_IF([test "x$KERNEL_FOUND" = "xno"], [ AC_MSG_WARN([Kernel Distro seems to be neither RedHat, SuSE, openEuler nor Ubuntu]) diff --git a/contrib/lbuild/funcs.sh b/contrib/lbuild/funcs.sh -index 78389a5d96..7889ff072e 100644 +index 3261a4752f..8cbe00b5a8 100644 --- a/contrib/lbuild/funcs.sh +++ b/contrib/lbuild/funcs.sh -@@ -221,6 +221,7 @@ autodetect_target() { - sles15.3) target="$(uname -r | cut -d . -f 1,2)-sles15sp3";; +@@ -223,6 +223,7 @@ autodetect_target() { sles15.4) target="$(uname -r | cut -d . -f 1,2)-sles15sp4";; + sles15.5) target="$(uname -r | cut -d . -f 1,2)-sles15sp5";; fc18) target="3.x-fc18";; + oe2003.sp3) target="4.19-oe2003sp3";; oe2203) target="5.10-oe2203";; @@ -2449,10 +2449,10 @@ index 0000000000..ec19e1e90f +rhel8/ext4-old_ea_inodes_handling_fix.patch +rhel8.4/ext4-optimize-find_delayed_extent.patch diff --git a/lustre/ChangeLog b/lustre/ChangeLog -index 3f03823e3d..6706ff15b1 100644 +index 7506abaaf1..d535470a3c 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog -@@ -100,6 +100,7 @@ TBD Whamcloud +@@ -104,6 +104,7 @@ vanilla linux 5.4.0 (ZFS + ldiskfs) vanilla linux 5.4.21 (ZFS + ldiskfs) vanilla linux 5.4.136 (ZFS + ldiskfs) @@ -2460,7 +2460,7 @@ index 3f03823e3d..6706ff15b1 100644 5.10.0-60.94.0.118.oe2203 (openEuler 22.03 LTS) 5.10.0-136.32.0.108.oe2203sp1 (openEuler 22.03 LTS SP1) 5.10.0-153.19.0.95.oe2203sp2 (openEuler 22.03 LTS SP2) -@@ -209,6 +210,7 @@ TBD Whamcloud +@@ -213,6 +214,7 @@ 5.8.0-53 (Ubuntu 20.04.2 HWE) 5.11.0-31 (Ubuntu 20.04.3 HWE) 5.11.0 (vanilla kernel.org) @@ -2469,12 +2469,12 @@ index 3f03823e3d..6706ff15b1 100644 * Recommended e2fsprogs version: 1.46.5.wc1 or newer * Recommended ZFS version: 2.1.5 diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 -index feed23e01f..251c977ea4 100644 +index 4657ebe337..4bb87dadaa 100644 --- a/lustre/autoconf/lustre-core.m4 +++ b/lustre/autoconf/lustre-core.m4 -@@ -3496,6 +3496,7 @@ lustre/kernel_patches/targets/5.3-sles15sp2.target - lustre/kernel_patches/targets/5.3-sles15sp3.target +@@ -4474,6 +4474,7 @@ lustre/kernel_patches/targets/5.3-sles15sp3.target lustre/kernel_patches/targets/5.14-sles15sp4.target + lustre/kernel_patches/targets/5.14-sles15sp5.target lustre/kernel_patches/targets/3.x-fc18.target +lustre/kernel_patches/targets/4.19-oe2003sp3.target lustre/kernel_patches/targets/5.10-oe2203.target diff --git a/0037-LU-16662-autoconf-fix-configure-test-compile-for-CON.patch b/0037-LU-16662-autoconf-fix-configure-test-compile-for-CON.patch deleted file mode 100644 index 0f3224cb3461401121ddebef9a2099cffdfed0bf..0000000000000000000000000000000000000000 --- a/0037-LU-16662-autoconf-fix-configure-test-compile-for-CON.patch +++ /dev/null @@ -1,76 +0,0 @@ -From da20815eeddf748ec815071fec8a786f1bc7c37b Mon Sep 17 00:00:00 2001 -From: Xinliang Liu -Date: Thu, 6 Apr 2023 23:24:04 -0700 -Subject: [PATCH 37/61] LU-16662 autoconf: fix configure test compile for - CONFIG_KEYS - -This fixes below configure error on Linux v5.19+: -$ ./configure --disable-server -... -checking whether to enable gss keyring backend... yes -checking if Linux kernel was built with CONFIG_KEYS in or as module... -no -configure: WARNING: GSS keyring backend requires that CONFIG_KEYS be -enabled in your kernel. -checking for keyctl_search in -lkeyutils... yes -configure: error: Cannot enable gss_keyring. See above for details. -$ grep CONFIG_KEYS -rn /boot/config-* -6884:CONFIG_KEYS=y - -For in-tree IB support and without passing Linux src path when run -./configure, the LINUX_OBJ maybe just a soft link to O2IBPATH, they -are pointing to the same dir. E.g.: -O2IBPATH='/usr/src/kernels/6.1.8-3.0.0.7.oe1.aarch64' -LINUX_OBJ='/lib/modules/6.1.8-3.0.0.7.oe1.aarch64/build' -$ ls -l /lib/modules/6.1.8-3.0.0.7.oe1.aarch64/build -lrwxrwxrwx 1 root root 42 Feb 7 00:00 -/lib/modules/6.1.8-3.0.0.7.oe1.aarch64/build -> -/usr/src/kernels/6.1.8-3.0.0.7.oe1.aarch64 -In this case, current configure will put kernel's Module.symvers to -variable KBUILD_EXTRA_SYMBOLS. This should be avoided after kernel -v5.19 which contains commit "b8422711080f modpost: make multiple export -error". This making multiple export symbol as an error from a warning -which can be seen in the config.log: -... -ERROR: modpost: vmlinux: 'init_uts_ns' exported twice. Previous export -was in vmlinux -... - -Lustre-change: https://review.whamcloud.com/50399 -Lustre-commit: 321a533b868908f37d01a4b787f5a463a02e427c - -Test-Parameters: trivial -Change-Id: I35295b3acc7fffb93716362f5d8c659eb922afcb -Signed-off-by: Xinliang Liu -Reviewed-by: Shaun Tancheff -Reviewed-by: James Simmons -Signed-off-by: Xinliang Liu ---- - lnet/autoconf/lustre-lnet.m4 | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 -index da6fd582b2..f1dce43dd3 100644 ---- a/lnet/autoconf/lustre-lnet.m4 -+++ b/lnet/autoconf/lustre-lnet.m4 -@@ -251,7 +251,7 @@ AS_IF([test $ENABLEO2IB = "no"], [ - O2IB_SYMVER=$LINUX_OBJ/Module.symvers - fi - if test -n "$O2IB_SYMVER"; then -- if test "$O2IB_SYMVER" != "$LINUX_OBJ/Module.symvers"; then -+ if test ! "$O2IB_SYMVER" -ef "$LINUX_OBJ/Module.symvers"; then - AC_MSG_NOTICE([adding $O2IB_SYMVER to Symbol Path O2IB]) - EXTRA_SYMBOLS="$EXTRA_SYMBOLS $O2IB_SYMVER" - AC_SUBST(EXTRA_SYMBOLS) -@@ -292,7 +292,7 @@ AC_SUBST(ENABLEO2IB) - - AS_IF([test $ENABLEO2IB != "no"], [ - EXTRA_CHECK_INCLUDE="$EXTRA_OFED_CONFIG $EXTRA_OFED_INCLUDE" -- if test $O2IBPATH != $LINUX_OBJ; then -+ if test ! $O2IBPATH -ef $LINUX_OBJ; then - KBUILD_EXTRA_SYMBOLS="$KBUILD_EXTRA_SYMBOLS $O2IBPATH/Module.symvers" - fi - --- -2.33.0 - diff --git a/0047-lustre.spec.in-Add-gcc-option-Wno-stringop-overflow.patch b/0037-lustre.spec.in-Add-gcc-option-Wno-stringop-overflow.patch similarity index 76% rename from 0047-lustre.spec.in-Add-gcc-option-Wno-stringop-overflow.patch rename to 0037-lustre.spec.in-Add-gcc-option-Wno-stringop-overflow.patch index 9e4f6fad5933055e9a32dce82d8ec47e6dcc7ad8..21e7e5d3d53a9fb960e8157919635e50f5254ad7 100644 --- a/0047-lustre.spec.in-Add-gcc-option-Wno-stringop-overflow.patch +++ b/0037-lustre.spec.in-Add-gcc-option-Wno-stringop-overflow.patch @@ -1,7 +1,7 @@ -From 9021e6e81632d116c9bc0e5d59ce2b1a47cc1559 Mon Sep 17 00:00:00 2001 +From 79b9290d0f58fbc21e98bb2329fabe6d4d209823 Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Fri, 24 Mar 2023 08:21:06 +0000 -Subject: [PATCH 47/61] lustre.spec.in: Add gcc option -Wno-stringop-overflow +Subject: [PATCH 37/49] lustre.spec.in: Add gcc option -Wno-stringop-overflow See openEuler gcc bug: https://gitee.com/openeuler/gcc/issues/I5XMD0 @@ -11,10 +11,10 @@ Signed-off-by: Xinliang Liu 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lustre.spec.in b/lustre.spec.in -index 9aa271f504..8f9882f989 100644 +index 82fec51ab2..9f394956bc 100644 --- a/lustre.spec.in +++ b/lustre.spec.in -@@ -472,7 +472,7 @@ export UTILS_CFLAGS="${UTILS_CFLAGS} -D__SANE_USERSPACE_TYPES__=1" +@@ -476,7 +476,7 @@ export UTILS_CFLAGS="${UTILS_CFLAGS} -D__SANE_USERSPACE_TYPES__=1" # Disable any hardening or annotation since this doesn't make sense for # kernel code, and reset "optflags" so that the vendor's overzealous flags don't # create build failures. diff --git a/0048-LU-16321-osd-Allow-fiemap-on-kernel-buffers.patch b/0038-LU-16321-osd-Allow-fiemap-on-kernel-buffers.patch similarity index 97% rename from 0048-LU-16321-osd-Allow-fiemap-on-kernel-buffers.patch rename to 0038-LU-16321-osd-Allow-fiemap-on-kernel-buffers.patch index d37d58edd91b60c145dbf21704d88c5ccf24a9cd..a05c230f3fe85367fde6dbc082eacf657ad34e5c 100644 --- a/0048-LU-16321-osd-Allow-fiemap-on-kernel-buffers.patch +++ b/0038-LU-16321-osd-Allow-fiemap-on-kernel-buffers.patch @@ -1,7 +1,7 @@ -From e55361ee38116437b8b6529368f6a25160bfa410 Mon Sep 17 00:00:00 2001 +From 0e0d0f03dbfe92facbf0660dee8ba7250503cd43 Mon Sep 17 00:00:00 2001 From: Shaun Tancheff Date: Fri, 2 Dec 2022 04:19:59 -0600 -Subject: [PATCH 48/61] LU-16321 osd: Allow fiemap on kernel buffers +Subject: [PATCH 38/49] LU-16321 osd: Allow fiemap on kernel buffers Linux commit v5.17-rc3-19-g967747bbc084 uaccess: remove CONFIG_SET_FS @@ -385,10 +385,10 @@ index 1cc20c8e58..82711ec7a5 100644 +linux-5.10/ext4-fiemap-kernel-data.patch rhel8/ext4-old_ea_inodes_handling_fix.patch diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c -index 233d88397c..19b5da7bd1 100644 +index 1af570e932..9657b60379 100644 --- a/lustre/osd-ldiskfs/osd_io.c +++ b/lustre/osd-ldiskfs/osd_io.c -@@ -1357,6 +1357,22 @@ struct osd_fextent { +@@ -1368,6 +1368,22 @@ struct osd_fextent { unsigned int mapped:1; }; @@ -411,7 +411,7 @@ index 233d88397c..19b5da7bd1 100644 static int osd_is_mapped(struct dt_object *dt, __u64 offset, struct osd_fextent *cached_extent) { -@@ -1366,6 +1382,7 @@ static int osd_is_mapped(struct dt_object *dt, __u64 offset, +@@ -1377,6 +1393,7 @@ static int osd_is_mapped(struct dt_object *dt, __u64 offset, struct fiemap_extent_info fei = { 0 }; struct fiemap_extent fe = { 0 }; int rc; @@ -419,7 +419,7 @@ index 233d88397c..19b5da7bd1 100644 if (block >= cached_extent->start && block < cached_extent->end) return cached_extent->mapped; -@@ -1379,8 +1396,9 @@ static int osd_is_mapped(struct dt_object *dt, __u64 offset, +@@ -1390,8 +1407,9 @@ static int osd_is_mapped(struct dt_object *dt, __u64 offset, fei.fi_extents_max = 1; fei.fi_extents_start = &fe; @@ -430,7 +430,7 @@ index 233d88397c..19b5da7bd1 100644 if (rc != 0) return 0; -@@ -2633,6 +2651,7 @@ static int osd_fiemap_get(const struct lu_env *env, struct dt_object *dt, +@@ -2650,6 +2668,7 @@ static int osd_fiemap_get(const struct lu_env *env, struct dt_object *dt, struct inode *inode = osd_dt_obj(dt)->oo_inode; u64 len; int rc; @@ -438,7 +438,7 @@ index 233d88397c..19b5da7bd1 100644 LASSERT(inode); if (inode->i_op->fiemap == NULL) -@@ -2652,7 +2671,9 @@ static int osd_fiemap_get(const struct lu_env *env, struct dt_object *dt, +@@ -2669,7 +2688,9 @@ static int osd_fiemap_get(const struct lu_env *env, struct dt_object *dt, if (fieinfo.fi_flags & FIEMAP_FLAG_SYNC) filemap_write_and_wait(inode->i_mapping); diff --git a/0050-LU-16893-libcfs-Remove-force_sig-usage-from-lfsck.patch b/0039-LU-16893-libcfs-Remove-force_sig-usage-from-lfsck.patch similarity index 95% rename from 0050-LU-16893-libcfs-Remove-force_sig-usage-from-lfsck.patch rename to 0039-LU-16893-libcfs-Remove-force_sig-usage-from-lfsck.patch index 0ad2da1c62721192f1faa210c35afb2c8c624a19..3db00e6f94ad48b98970cceb648708e2e5209b3b 100644 --- a/0050-LU-16893-libcfs-Remove-force_sig-usage-from-lfsck.patch +++ b/0039-LU-16893-libcfs-Remove-force_sig-usage-from-lfsck.patch @@ -1,7 +1,7 @@ -From df7fe641427f384e33d06cb837b09af59dd2830e Mon Sep 17 00:00:00 2001 +From a3fcbfe5afbe1473e17fc7ca7cee54cf35feeb7d Mon Sep 17 00:00:00 2001 From: Shaun Tancheff Date: Tue, 27 Jun 2023 15:10:19 +0700 -Subject: [PATCH 50/61] LU-16893 libcfs: Remove force_sig usage from lfsck +Subject: [PATCH 39/49] LU-16893 libcfs: Remove force_sig usage from lfsck The lfsck pool of kernel threads uses force_sig() to signal the worker threads to stop. A signal is used here as the @@ -31,7 +31,7 @@ Signed-off-by: Xinliang Liu 4 files changed, 16 insertions(+), 43 deletions(-) diff --git a/libcfs/autoconf/lustre-libcfs.m4 b/libcfs/autoconf/lustre-libcfs.m4 -index 2e99cff179..34726fe8dc 100644 +index 37c2762cbd..ec169c9b28 100644 --- a/libcfs/autoconf/lustre-libcfs.m4 +++ b/libcfs/autoconf/lustre-libcfs.m4 @@ -1742,27 +1742,6 @@ AC_DEFUN([LIBCFS_LOOKUP_USER_KEY], [ @@ -62,7 +62,7 @@ index 2e99cff179..34726fe8dc 100644 # # LIBCFS_CACHE_DETAIL_WRITERS # -@@ -2305,7 +2284,6 @@ AC_DEFUN([LIBCFS_PROG_LINUX_SRC], [ +@@ -2357,7 +2336,6 @@ AC_DEFUN([LIBCFS_PROG_LINUX_SRC], [ LIBCFS_SRC_KOBJ_TYPE_DEFAULT_GROUPS # 5.3 LIBCFS_SRC_LOOKUP_USER_KEY @@ -70,7 +70,7 @@ index 2e99cff179..34726fe8dc 100644 LIBCFS_SRC_CACHE_DETAIL_WRITERS # 5.4 LIBCFS_SRC_GENL_DUMPIT_INFO -@@ -2444,7 +2422,6 @@ AC_DEFUN([LIBCFS_PROG_LINUX_RESULTS], [ +@@ -2499,7 +2477,6 @@ AC_DEFUN([LIBCFS_PROG_LINUX_RESULTS], [ LIBCFS_KOBJ_TYPE_DEFAULT_GROUPS # 5.3 LIBCFS_LOOKUP_USER_KEY diff --git a/0051-LU-16534-build-Prefer-timer_delete-_sync.patch b/0040-LU-16534-build-Prefer-timer_delete-_sync.patch similarity index 82% rename from 0051-LU-16534-build-Prefer-timer_delete-_sync.patch rename to 0040-LU-16534-build-Prefer-timer_delete-_sync.patch index 5733d496dd548d6d3f0a16d68fec21db48e0190b..8f0dbeea2a57b528c355601e15a65fe40da30b7e 100644 --- a/0051-LU-16534-build-Prefer-timer_delete-_sync.patch +++ b/0040-LU-16534-build-Prefer-timer_delete-_sync.patch @@ -1,7 +1,7 @@ -From 929ab1c506f1f46a1860422e084545734819301a Mon Sep 17 00:00:00 2001 +From ae98bfde91a64f834f873dcd6b88e227c6181822 Mon Sep 17 00:00:00 2001 From: Shaun Tancheff Date: Tue, 7 Feb 2023 02:18:36 -0600 -Subject: [PATCH 51/61] LU-16534 build: Prefer timer_delete[_sync] +Subject: [PATCH 40/49] LU-16534 build: Prefer timer_delete[_sync] Linux commit v6.1-rc1-7-g9a5a30568697 timers: Get rid of del_singleshot_timer_sync() @@ -34,170 +34,10 @@ Signed-off-by: Xinliang Liu libcfs/autoconf/lustre-libcfs.m4 lnet/klnds/kfilnd/kfilnd_tn.c --- - contrib/scripts/spelling.txt | 3 + - libcfs/autoconf/lustre-libcfs.m4 | 58 ++ - lnet/klnds/gnilnd/gnilnd_cb.c | 8 +- - lnet/klnds/gnilnd/gnilnd_conn.c | 2 +- - lnet/klnds/kfilnd/kfilnd_tn.c | 1646 ++++++++++++++++++++++++++++++ - lnet/lnet/net_fault.c | 4 +- - lustre/ldlm/ldlm_lockd.c | 2 +- - lustre/lod/lod_qos.c | 2 +- - lustre/osp/osp_precreate.c | 6 +- - lustre/ptlrpc/gss/gss_keyring.c | 2 +- - lustre/ptlrpc/service.c | 6 +- - 11 files changed, 1723 insertions(+), 16 deletions(-) + lnet/klnds/kfilnd/kfilnd_tn.c | 1646 +++++++++++++++++++++++++++++++++ + 1 file changed, 1646 insertions(+) create mode 100644 lnet/klnds/kfilnd/kfilnd_tn.c -diff --git a/contrib/scripts/spelling.txt b/contrib/scripts/spelling.txt -index b812339ea3..5e34d04572 100644 ---- a/contrib/scripts/spelling.txt -+++ b/contrib/scripts/spelling.txt -@@ -100,6 +100,9 @@ cfs_time_current_sec||ktime_get_real_seconds - CLASSERT||BUILD_BUG_ON() - msecs_to_jiffies||cfs_time_seconds - DEFINE_TIMER||CFS_DEFINE_TIMER -+del_timer||timer_delete -+del_timer_sync||timer_delete_sync -+del_singleshot_timer_sync||timer_delete_sync - EWOULDBLOCK||EAGAIN - container_of0||container_of_safe - DN_MAX_BONUSLEN||DN_BONUS_SIZE(dnodesize) -diff --git a/libcfs/autoconf/lustre-libcfs.m4 b/libcfs/autoconf/lustre-libcfs.m4 -index 34726fe8dc..ec169c9b28 100644 ---- a/libcfs/autoconf/lustre-libcfs.m4 -+++ b/libcfs/autoconf/lustre-libcfs.m4 -@@ -2176,6 +2176,58 @@ AC_DEFUN([LIBCFS_PDE_DATA_EXISTS],[ - ]) - ]) # LIBCFS_PDE_DATA_EXISTS - -+# -+# LIBCFS_TIMER_DELETE_SYNC -+# -+# Linux commit v6.1-rc1-7-g9a5a30568697 -+# timers: Get rid of del_singleshot_timer_sync() -+# Linux commit v6.1-rc1-11-g9b13df3fb64e -+# timers: Rename del_timer_sync() to timer_delete_sync() -+# -+AC_DEFUN([LIBCFS_SRC_TIMER_DELETE_SYNC],[ -+ LB2_LINUX_TEST_SRC([timer_delete_sync], [ -+ #include -+ ],[ -+ struct timer_list *timer = NULL; -+ (void)timer_delete_sync(timer); -+ ],[]) -+]) -+AC_DEFUN([LIBCFS_TIMER_DELETE_SYNC],[ -+ AC_MSG_CHECKING([is timer_delete_sync() available]) -+ LB2_LINUX_TEST_RESULT([timer_delete_sync], [ -+ AC_DEFINE(HAVE_TIMER_DELETE_SYNC, 1, -+ [timer_delete_sync() is available]) -+ ],[ -+ AC_DEFINE(timer_delete_sync(t), del_timer_sync(t), -+ [timer_delete_sync() not is available]) -+ ]) -+]) # LIBCFS_TIMER_DELETE_SYNC -+ -+# -+# LIBCFS_TIMER_DELETE_SYNC -+# -+# Linux commit v6.1-rc1-12-gbb663f0f3c39 -+# timers: Rename del_timer() to timer_delete() -+# -+AC_DEFUN([LIBCFS_SRC_TIMER_DELETE],[ -+ LB2_LINUX_TEST_SRC([timer_delete], [ -+ #include -+ ],[ -+ struct timer_list *timer = NULL; -+ (void)timer_delete(timer); -+ ],[]) -+]) -+AC_DEFUN([LIBCFS_TIMER_DELETE],[ -+ AC_MSG_CHECKING([is timer_delete() available]) -+ LB2_LINUX_TEST_RESULT([timer_delete], [ -+ AC_DEFINE(HAVE_TIMER_DELETE, 1, -+ [timer_delete() is available]) -+ ],[ -+ AC_DEFINE(timer_delete(t), del_timer(t), -+ [timer_delete() not is available]) -+ ]) -+]) # LIBCFS_TIMER_DELETE -+ - dnl # - dnl # Generate and compile all of the kernel API test cases to determine - dnl # which interfaces are available. By invoking the kernel build system -@@ -2312,6 +2364,9 @@ AC_DEFUN([LIBCFS_PROG_LINUX_SRC], [ - LIBCFS_SRC_PARAM_SET_UINT_MINMAX - # 5.17 - LIBCFS_SRC_PDE_DATA_EXISTS -+ # 6.2 -+ LIBCFS_SRC_TIMER_DELETE_SYNC -+ LIBCFS_SRC_TIMER_DELETE - - LB2_LINUX_TEST_COMPILE_ALL([libcfs], - [for available kernel interfaces to libcfs]) -@@ -2450,6 +2505,9 @@ AC_DEFUN([LIBCFS_PROG_LINUX_RESULTS], [ - LIBCFS_PARAM_SET_UINT_MINMAX - # 5.17 - LIBCFS_PDE_DATA_EXISTS -+ # 6.2 -+ LIBCFS_TIMER_DELETE_SYNC -+ LIBCFS_TIMER_DELETE - ]) - - # -diff --git a/lnet/klnds/gnilnd/gnilnd_cb.c b/lnet/klnds/gnilnd/gnilnd_cb.c -index 6be0b2a74b..abf88d3928 100644 ---- a/lnet/klnds/gnilnd/gnilnd_cb.c -+++ b/lnet/klnds/gnilnd/gnilnd_cb.c -@@ -2988,7 +2988,7 @@ kgnilnd_reaper(void *arg) - CDEBUG(D_INFO, "awake after schedule\n"); - } - -- del_singleshot_timer_sync(&timer); -+ timer_delete_sync(&timer); - spin_lock(&kgnilnd_data.kgn_reaper_lock); - finish_wait(&kgnilnd_data.kgn_reaper_waitq, &wait); - continue; -@@ -3805,7 +3805,7 @@ kgnilnd_process_rdmaq(kgn_device_t *dev) - /* if we think we need to adjust, take lock to serialize and recheck */ - spin_lock(&dev->gnd_rdmaq_lock); - if (time_after_eq(jiffies, dev->gnd_rdmaq_deadline)) { -- del_singleshot_timer_sync(&dev->gnd_rdmaq_timer); -+ timer_delete_sync(&dev->gnd_rdmaq_timer); - - dead_bump = cfs_time_seconds(1) / *kgnilnd_tunables.kgn_rdmaq_intervals; - -@@ -4636,7 +4636,7 @@ kgnilnd_process_mapped_tx(kgn_device_t *dev) - spin_lock(&dev->gnd_lock); - if (list_empty(&dev->gnd_map_tx)) { - /* if the list is empty make sure we dont have a timer running */ -- del_singleshot_timer_sync(&dev->gnd_map_timer); -+ timer_delete_sync(&dev->gnd_map_timer); - spin_unlock(&dev->gnd_lock); - RETURN(0); - } -@@ -4663,7 +4663,7 @@ kgnilnd_process_mapped_tx(kgn_device_t *dev) - } - - /* delete the previous timer if it exists */ -- del_singleshot_timer_sync(&dev->gnd_map_timer); -+ timer_delete_sync(&dev->gnd_map_timer); - /* stash the last map version to let us know when a good one was seen */ - last_map_version = dev->gnd_map_version; - -diff --git a/lnet/klnds/gnilnd/gnilnd_conn.c b/lnet/klnds/gnilnd/gnilnd_conn.c -index 38dce1d475..9ac954957f 100644 ---- a/lnet/klnds/gnilnd/gnilnd_conn.c -+++ b/lnet/klnds/gnilnd/gnilnd_conn.c -@@ -2545,7 +2545,7 @@ kgnilnd_dgram_mover(void *arg) - deadline = jiffies + cfs_time_seconds(*kgnilnd_tunables.kgn_dgram_timeout); - } - -- del_singleshot_timer_sync(&timer.timer); -+ timer_delete_sync(&timer.timer); - finish_wait(&dev->gnd_dgram_waitq, &wait); - } - diff --git a/lnet/klnds/kfilnd/kfilnd_tn.c b/lnet/klnds/kfilnd/kfilnd_tn.c new file mode 100644 index 0000000000..59fe030fb4 @@ -1850,129 +1690,6 @@ index 0000000000..59fe030fb4 + + return 0; +} -diff --git a/lnet/lnet/net_fault.c b/lnet/lnet/net_fault.c -index 0a201a0b3c..98bdb1664f 100644 ---- a/lnet/lnet/net_fault.c -+++ b/lnet/lnet/net_fault.c -@@ -649,7 +649,7 @@ delayed_msg_check(struct lnet_delay_rule *rule, bool all, - } - - if (list_empty(&rule->dl_msg_list)) { -- del_timer(&rule->dl_timer); -+ timer_delete(&rule->dl_timer); - rule->dl_msg_send = -1; - - } else if (!list_empty(msg_list)) { -@@ -936,7 +936,7 @@ lnet_delay_rule_del(lnet_nid_t src, lnet_nid_t dst, bool shutdown) - list_for_each_entry_safe(rule, tmp, &rule_list, dl_link) { - list_del_init(&rule->dl_link); - -- del_timer_sync(&rule->dl_timer); -+ timer_delete_sync(&rule->dl_timer); - delayed_msg_check(rule, true, &msg_list); - delay_rule_decref(rule); /* -1 for the_lnet.ln_delay_rules */ - n++; -diff --git a/lustre/ldlm/ldlm_lockd.c b/lustre/ldlm/ldlm_lockd.c -index 7e59709ea6..7a42d5d3df 100644 ---- a/lustre/ldlm/ldlm_lockd.c -+++ b/lustre/ldlm/ldlm_lockd.c -@@ -511,7 +511,7 @@ static int __ldlm_del_waiting_lock(struct ldlm_lock *lock) - /* Removing the head of the list, adjust timer. */ - if (list_next == &waiting_locks_list) { - /* No more, just cancel. */ -- del_timer(&waiting_locks_timer); -+ timer_delete(&waiting_locks_timer); - } else { - time64_t now = ktime_get_seconds(); - struct ldlm_lock *next; -diff --git a/lustre/lod/lod_qos.c b/lustre/lod/lod_qos.c -index 6447dd90db..a4ba6786b0 100644 ---- a/lustre/lod/lod_qos.c -+++ b/lustre/lod/lod_qos.c -@@ -1488,7 +1488,7 @@ static int lod_ost_alloc_qos(const struct lu_env *env, struct lod_object *lo, - /* Do actual allocation, use write lock here. */ - rc = down_write_killable(&lod->lod_ost_descs.ltd_qos.lq_rw_sem); - -- del_singleshot_timer_sync(&timer.timer); -+ timer_delete_sync(&timer.timer); - kernel_sigaction(SIGKILL, SIG_IGN); - if (rc) { - flush_signals(current); -diff --git a/lustre/osp/osp_precreate.c b/lustre/osp/osp_precreate.c -index 5585ce1bf4..d9d64ff536 100644 ---- a/lustre/osp/osp_precreate.c -+++ b/lustre/osp/osp_precreate.c -@@ -239,7 +239,7 @@ static int osp_statfs_update(const struct lu_env *env, struct osp_device *d) - /* - * no updates till reply - */ -- del_timer(&d->opd_statfs_timer); -+ timer_delete(&d->opd_statfs_timer); - d->opd_statfs_fresh_till = ktime_add_ns(ktime_get(), expire); - d->opd_statfs_update_in_progress = 1; - -@@ -293,7 +293,7 @@ void osp_statfs_need_now(struct osp_device *d) - * is replied - */ - d->opd_statfs_fresh_till = ktime_sub_ns(ktime_get(), NSEC_PER_SEC); -- del_timer(&d->opd_statfs_timer); -+ timer_delete(&d->opd_statfs_timer); - wake_up(&d->opd_pre_waitq); - } - } -@@ -1837,7 +1837,7 @@ void osp_statfs_fini(struct osp_device *d) - struct task_struct *task = d->opd_pre_task; - ENTRY; - -- del_timer(&d->opd_statfs_timer); -+ timer_delete(&d->opd_statfs_timer); - - d->opd_pre_task = NULL; - if (task) -diff --git a/lustre/ptlrpc/gss/gss_keyring.c b/lustre/ptlrpc/gss/gss_keyring.c -index 124ebe1dc1..5d4e3c19a0 100644 ---- a/lustre/ptlrpc/gss/gss_keyring.c -+++ b/lustre/ptlrpc/gss/gss_keyring.c -@@ -149,7 +149,7 @@ void ctx_clear_timer_kr(struct ptlrpc_cli_ctx *ctx) - - CDEBUG(D_SEC, "ctx %p, key %p\n", ctx, gctx_kr->gck_key); - -- del_singleshot_timer_sync(timer); -+ timer_delete_sync(timer); - } - - static -diff --git a/lustre/ptlrpc/service.c b/lustre/ptlrpc/service.c -index 16fcaba8f0..366dc5a20b 100644 ---- a/lustre/ptlrpc/service.c -+++ b/lustre/ptlrpc/service.c -@@ -1248,7 +1248,7 @@ static void ptlrpc_at_set_timer(struct ptlrpc_service_part *svcpt) - time64_t next; - - if (array->paa_count == 0) { -- del_timer(&svcpt->scp_at_timer); -+ timer_delete(&svcpt->scp_at_timer); - return; - } - -@@ -3390,7 +3390,7 @@ ptlrpc_service_del_atimer(struct ptlrpc_service *svc) - /* early disarm AT timer... */ - ptlrpc_service_for_each_part(svcpt, i, svc) { - if (svcpt->scp_service != NULL) -- del_timer(&svcpt->scp_at_timer); -+ timer_delete(&svcpt->scp_at_timer); - } - } - -@@ -3558,7 +3558,7 @@ ptlrpc_service_free(struct ptlrpc_service *svc) - break; - - /* In case somebody rearmed this in the meantime */ -- del_timer(&svcpt->scp_at_timer); -+ timer_delete(&svcpt->scp_at_timer); - array = &svcpt->scp_at_array; - - if (array->paa_reqs_array != NULL) { -- 2.33.0 diff --git a/0053-LU-16788-tests-sanity-should-remove-temp-files.patch b/0041-LU-16788-tests-sanity-should-remove-temp-files.patch similarity index 84% rename from 0053-LU-16788-tests-sanity-should-remove-temp-files.patch rename to 0041-LU-16788-tests-sanity-should-remove-temp-files.patch index 888acce0f7bed8d38e7b0060bbc18e5b1e6e92a9..231f3237cd0b8819b133393d9e701b4ed90507ef 100644 --- a/0053-LU-16788-tests-sanity-should-remove-temp-files.patch +++ b/0041-LU-16788-tests-sanity-should-remove-temp-files.patch @@ -1,7 +1,7 @@ -From bc58052e3b1a1883e4a22eccdd43a3992b99ec42 Mon Sep 17 00:00:00 2001 +From 984d0ef8cc1ca664c1e33994bb148c8adea67974 Mon Sep 17 00:00:00 2001 From: Alex Zhuravlev Date: Mon, 1 May 2023 18:05:35 +0300 -Subject: [PATCH 53/61] LU-16788 tests: sanity should remove temp files +Subject: [PATCH 41/49] LU-16788 tests: sanity should remove temp files during the test to fit OSTSIZE @@ -20,10 +20,10 @@ Signed-off-by: Xinliang Liu 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh -index c2693abd2d..a004738552 100755 +index ca0ea72ade..7c05ad48bc 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh -@@ -7094,6 +7094,7 @@ test_56w() { +@@ -7089,6 +7089,7 @@ test_56w() { local dir=$DIR/$tdir setup_56 $dir $NUMFILES $NUMDIRS "-c $OSTCOUNT" "-c1" @@ -31,7 +31,7 @@ index c2693abd2d..a004738552 100755 local stripe_size=$($LFS getstripe -S -d $dir) || error "$LFS getstripe -S -d $dir failed" -@@ -7695,6 +7696,7 @@ test_56xd() { +@@ -7690,6 +7691,7 @@ test_56xd() { local layout_after; test_mkdir "$dir" || error "cannot create dir $dir" @@ -39,7 +39,7 @@ index c2693abd2d..a004738552 100755 $LFS setstripe $layout_yaml $f_yaml || error "cannot setstripe $f_yaml with layout $layout_yaml" $LFS getstripe --yaml $f_yaml > $yamlfile -@@ -7731,6 +7733,7 @@ test_56xe() { +@@ -7726,6 +7728,7 @@ test_56xe() { local layout_after="" test_mkdir "$dir" || error "cannot create dir $dir" @@ -47,7 +47,7 @@ index c2693abd2d..a004738552 100755 $LFS setstripe $layout $f_comp || error "cannot setstripe $f_comp with layout $layout" layout_before=$(get_layout_param $f_comp) -@@ -7763,6 +7766,7 @@ test_56xf() { +@@ -7758,6 +7761,7 @@ test_56xf() { local fid_after="" test_mkdir "$dir" || error "cannot create dir $dir" @@ -55,7 +55,7 @@ index c2693abd2d..a004738552 100755 $LFS setstripe $layout $f_comp || error "cannot setstripe $f_comp with layout $layout" fid_before=$($LFS getstripe --fid $f_comp) -@@ -7847,6 +7851,7 @@ test_56xg() { +@@ -7842,6 +7846,7 @@ test_56xg() { # init the file to migrate $LFS setstripe -c1 -i1 $DIR/$tfile || error "Unable to create $tfile on OST1" @@ -63,7 +63,7 @@ index c2693abd2d..a004738552 100755 dd if=/dev/urandom of=$DIR/$tfile bs=1M count=4 status=none || error "Unable to write on $tfile" -@@ -8884,6 +8889,7 @@ test_64e() { +@@ -8905,6 +8910,7 @@ test_64e() { local grants=$((wb_round_up + extent_tax)) $LFS setstripe -c 1 -i 0 $DIR/$tfile || error "lfs setstripe failed" @@ -71,7 +71,7 @@ index c2693abd2d..a004738552 100755 # define OBD_FAIL_TGT_NO_GRANT 0x725 # make the server not grant more back -@@ -9042,6 +9048,7 @@ test_64h() { +@@ -9063,6 +9069,7 @@ test_64h() { $LCTL set_param osc.*OST0000*.grant_shrink_interval=10 $LFS setstripe -c 1 -i 0 $DIR/$tfile @@ -79,7 +79,7 @@ index c2693abd2d..a004738552 100755 dd if=/dev/zero of=$DIR/$tfile bs=1M count=10 oflag=sync # drop cache so that coming read would do rpc -@@ -9081,6 +9088,7 @@ test_64i() { +@@ -9102,6 +9109,7 @@ test_64i() { remote_ost_nodsh && skip "remote OSTs with nodsh" $LFS setstripe -c 1 -i 0 $DIR/$tfile @@ -87,7 +87,7 @@ index c2693abd2d..a004738552 100755 dd if=/dev/zero of=$DIR/$tfile bs=1M count=64 -@@ -9527,6 +9535,7 @@ test_69() { +@@ -9548,6 +9556,7 @@ test_69() { f="$DIR/$tfile" $LFS setstripe -c 1 -i 0 $f @@ -95,7 +95,7 @@ index c2693abd2d..a004738552 100755 $DIRECTIO write ${f}.2 0 1 || error "directio write error" -@@ -9544,7 +9553,6 @@ test_69() { +@@ -9565,7 +9574,6 @@ test_69() { $DIRECTIO read $f 1 1 && error "read succeeded, expect -ENOENT" do_facet ost1 lctl set_param fail_loc=0 @@ -103,7 +103,7 @@ index c2693abd2d..a004738552 100755 } run_test 69 "verify oa2dentry return -ENOENT doesn't LBUG ======" -@@ -11518,6 +11526,7 @@ test_103b() { +@@ -11546,6 +11554,7 @@ test_103b() { declare -a pids local U @@ -111,7 +111,7 @@ index c2693abd2d..a004738552 100755 for U in {0..511}; do { local O=$(printf "%04o" $U) -@@ -13697,6 +13706,7 @@ OLDIFS="$IFS" +@@ -13725,6 +13734,7 @@ OLDIFS="$IFS" cleanup_130() { trap 0 IFS="$OLDIFS" @@ -119,7 +119,7 @@ index c2693abd2d..a004738552 100755 } test_130a() { -@@ -13942,6 +13952,7 @@ test_130e() { +@@ -13970,6 +13980,7 @@ test_130e() { local fm_file=$DIR/$tfile $LFS setstripe -S 131072 -c 2 $fm_file || error "setstripe on $fm_file" @@ -127,7 +127,7 @@ index c2693abd2d..a004738552 100755 NUM_BLKS=512 EXPECTED_LEN=$(( (NUM_BLKS / 2) * 64 )) -@@ -14793,7 +14804,7 @@ test_150d() { +@@ -14835,7 +14846,7 @@ test_150d() { [[ "x$DOM" == "xyes" ]] && striping="-L mdt" @@ -136,7 +136,7 @@ index c2693abd2d..a004738552 100755 $LFS setstripe -E1M $striping -E eof -c $OSTCOUNT -S1M $DIR/$tdir || error "setstripe failed" fallocate -o 1G -l ${OSTCOUNT}m $DIR/$tdir || error "fallocate failed" -@@ -18305,14 +18316,12 @@ test_187a() { +@@ -18354,14 +18365,12 @@ test_187a() { local file=$dir0/file1 dd if=/dev/urandom of=$file count=10 bs=1M conv=fsync @@ -152,7 +152,7 @@ index c2693abd2d..a004738552 100755 } run_test 187a "Test data version change" -@@ -19331,6 +19340,7 @@ test_224c() { # LU-6441 +@@ -19380,6 +19389,7 @@ test_224c() { # LU-6441 #define OBD_FAIL_PTLRPC_CLIENT_BULK_CB3 0x520 do_facet ost1 "$LCTL set_param fail_loc=0x520" $LFS setstripe -c 1 -i 0 $DIR/$tfile @@ -160,7 +160,7 @@ index c2693abd2d..a004738552 100755 dd if=/dev/zero of=$DIR/$tfile bs=8MB count=1 sync do_facet ost1 "$LCTL set_param fail_loc=0" -@@ -20714,6 +20724,7 @@ test_231a() +@@ -20763,6 +20773,7 @@ test_231a() mkdir -p $DIR/$tdir $LFS setstripe -S ${brw_size}M $DIR/$tdir || error "failed to set stripe with -S ${brw_size}M option" @@ -168,7 +168,7 @@ index c2693abd2d..a004738552 100755 # clear the OSC stats $LCTL set_param osc.*.stats=0 &>/dev/null -@@ -20751,6 +20762,7 @@ run_test 231a "checking that reading/writing of BRW RPC size results in one RPC" +@@ -20800,6 +20811,7 @@ run_test 231a "checking that reading/writing of BRW RPC size results in one RPC" test_231b() { mkdir -p $DIR/$tdir @@ -176,7 +176,7 @@ index c2693abd2d..a004738552 100755 local i for i in {0..1023}; do dd if=/dev/zero of=$DIR/$tdir/$tfile conv=notrunc \ -@@ -20770,6 +20782,7 @@ test_232a() { +@@ -20819,6 +20831,7 @@ test_232a() { # ignore dd failure dd if=/dev/zero of=$DIR/$tdir/$tfile bs=1M count=1 || true @@ -184,7 +184,7 @@ index c2693abd2d..a004738552 100755 do_facet ost1 $LCTL set_param fail_loc=0 umount_client $MOUNT || error "umount failed" -@@ -20786,6 +20799,7 @@ test_232b() { +@@ -20835,6 +20848,7 @@ test_232b() { mkdir -p $DIR/$tdir $LFS setstripe -c1 -i0 $DIR/$tdir/$tfile dd if=/dev/zero of=$DIR/$tdir/$tfile bs=1M count=1 @@ -192,7 +192,7 @@ index c2693abd2d..a004738552 100755 sync cancel_lru_locks osc -@@ -21549,6 +21563,7 @@ test_253() { +@@ -21598,6 +21612,7 @@ test_253() { wait_mds_ost_sync wait_delete_completed mkdir $DIR/$tdir @@ -200,7 +200,7 @@ index c2693abd2d..a004738552 100755 pool_add $TESTNAME || error "Pool creation failed" pool_add_targets $TESTNAME 0 || error "Pool add targets failed" -@@ -22856,6 +22871,7 @@ test_272b() { +@@ -22905,6 +22920,7 @@ test_272b() { local dom=$DIR/$tdir/dom mkdir -p $DIR/$tdir $LFS setstripe -E 1M -L mdt -E -1 -c1 $dom @@ -208,7 +208,7 @@ index c2693abd2d..a004738552 100755 local mdtidx=$($LFS getstripe -m $dom) local mdtname=MDT$(printf %04x $mdtidx) -@@ -22898,6 +22914,7 @@ test_272c() { +@@ -22947,6 +22963,7 @@ test_272c() { local dom=$DIR/$tdir/$tfile mkdir -p $DIR/$tdir $LFS setstripe -E 1M -L mdt -E -1 -c1 $dom @@ -216,7 +216,7 @@ index c2693abd2d..a004738552 100755 local mdtidx=$($LFS getstripe -m $dom) local mdtname=MDT$(printf %04x $mdtidx) -@@ -23085,6 +23102,7 @@ test_275() { +@@ -23134,6 +23151,7 @@ test_275() { dd if=/dev/urandom of=$file bs=1M count=2 || error "failed to create a file" @@ -224,7 +224,7 @@ index c2693abd2d..a004738552 100755 cancel_lru_locks osc #lock 1 -@@ -24439,6 +24457,7 @@ test_398a() { # LU-4198 +@@ -24488,6 +24506,7 @@ test_398a() { # LU-4198 cut -d'.' -f2) $LFS setstripe -c 1 -i 0 $DIR/$tfile diff --git a/0057-LU-16571-utils-fix-parallel-lfs-migrate-b-on-hard-li.patch b/0042-LU-16571-utils-fix-parallel-lfs-migrate-b-on-hard-li.patch similarity index 93% rename from 0057-LU-16571-utils-fix-parallel-lfs-migrate-b-on-hard-li.patch rename to 0042-LU-16571-utils-fix-parallel-lfs-migrate-b-on-hard-li.patch index c6a519dc3ede4d4f2ffbc2c72061d4144511604e..7d8b2f51ed45e495a53274ba54ce69750f95a5a4 100644 --- a/0057-LU-16571-utils-fix-parallel-lfs-migrate-b-on-hard-li.patch +++ b/0042-LU-16571-utils-fix-parallel-lfs-migrate-b-on-hard-li.patch @@ -1,7 +1,7 @@ -From 6b25f2b394f1d1e83963fa9dcba5a40015e672a4 Mon Sep 17 00:00:00 2001 +From 0bf3984d6b2e844b8a550c2faceedddf1ef74d0f Mon Sep 17 00:00:00 2001 From: Etienne AUJAMES Date: Wed, 22 Feb 2023 11:37:49 +0100 -Subject: [PATCH 57/61] LU-16571 utils: fix parallel "lfs migrate -b" on hard +Subject: [PATCH 42/49] LU-16571 utils: fix parallel "lfs migrate -b" on hard links Multiple blocking "lfs migrate" on the same file can exhaust "ost" @@ -46,10 +46,10 @@ Reviewed-by: Patrick Farrell 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh -index 76f261b2d1..70f43a2dcd 100755 +index 7c05ad48bc..50cca26f41 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh -@@ -7874,6 +7874,44 @@ test_56xg() { +@@ -7869,6 +7869,44 @@ test_56xg() { } run_test 56xg "lfs migrate pool support" @@ -95,10 +95,10 @@ index 76f261b2d1..70f43a2dcd 100755 [ $MDS1_VERSION -lt $(version_code 2.4.53) ] && skip "No HSM $(lustre_build_version $SINGLEMDS) MDS < 2.4.53" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c -index 7946d9e050..4ae2a5f098 100644 +index ebe2fc823c..b952356087 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c -@@ -900,31 +900,34 @@ static int migrate_block(int fd, int fdv) +@@ -901,31 +901,34 @@ static int migrate_block(int fd, int fdv) int rc; int rc2; diff --git a/0060-Update-openEuler-22.03-kernels.patch b/0043-Update-openEuler-22.03-kernels.patch similarity index 99% rename from 0060-Update-openEuler-22.03-kernels.patch rename to 0043-Update-openEuler-22.03-kernels.patch index 12360f63573d08700242917a0ea9fe484aaefc9d..a15ecdcc62f24b45a2f8d904ad3f4c348e25ca39 100644 --- a/0060-Update-openEuler-22.03-kernels.patch +++ b/0043-Update-openEuler-22.03-kernels.patch @@ -1,7 +1,7 @@ -From 16169e4dfb7f3383ff62dc218c74aa8ff00dc0ba Mon Sep 17 00:00:00 2001 +From 8231e7be3822200f97dac5a80ac05364c744cbe3 Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Fri, 20 Oct 2023 10:12:18 +0000 -Subject: [PATCH 60/61] Update openEuler 22.03 kernels +Subject: [PATCH 43/49] Update openEuler 22.03 kernels Update the ldiskfs patch ext4-data-in-dirent.patch. Also fix kernel download URL for 22.03 LTS. diff --git a/0061-Update-kernel-for-openEuler-20.03-LTS.patch b/0044-Update-kernel-for-openEuler-20.03-LTS.patch similarity index 99% rename from 0061-Update-kernel-for-openEuler-20.03-LTS.patch rename to 0044-Update-kernel-for-openEuler-20.03-LTS.patch index 3b3f542077925cc342252e0b0c30131e14f480d5..fbd424981226ea0c005564959ac5b9ed3ef57942 100644 --- a/0061-Update-kernel-for-openEuler-20.03-LTS.patch +++ b/0044-Update-kernel-for-openEuler-20.03-LTS.patch @@ -1,7 +1,7 @@ -From 9f5b25ec2254a00041aa7f57c2df899c205c137e Mon Sep 17 00:00:00 2001 +From 0b31f4adf6d95055c16d33db28a3434e580c8b5e Mon Sep 17 00:00:00 2001 From: Xinliang Liu Date: Thu, 9 Nov 2023 02:25:34 +0000 -Subject: [PATCH 61/61] Update kernel for openEuler 20.03 LTS +Subject: [PATCH 44/49] Update kernel for openEuler 20.03 LTS Update ldiskfs patch ext4-data-in-dirent.patch. diff --git a/0045-ldiskfs-fix-build-for-oe2203sp3.patch b/0045-ldiskfs-fix-build-for-oe2203sp3.patch new file mode 100644 index 0000000000000000000000000000000000000000..dd36370a469d2457e55c4f0c0b245d9508b6f6b6 --- /dev/null +++ b/0045-ldiskfs-fix-build-for-oe2203sp3.patch @@ -0,0 +1,1151 @@ +From ea211b73f9706202ebda67cba91ac64de50543fe Mon Sep 17 00:00:00 2001 +From: Xinliang Liu +Date: Tue, 28 Nov 2023 07:53:08 +0000 +Subject: [PATCH 45/49] ldiskfs: fix build for oe2203sp3 + +Update ldiskfs patch for oe2203sp3 for new kernel +5.10.0-171.0.0.84.oe2203sp3. + +Signed-off-by: Xinliang Liu +--- + .../oe2203sp3/ext4-mballoc-extra-checks.patch | 320 ++++++++++++++ + .../patches/oe2203sp3/ext4-prealloc.patch | 404 ++++++++++++++++++ + .../oe2203sp3/ext4-simple-blockalloc.patch | 361 ++++++++++++++++ + .../series/ldiskfs-5.10.0-oe2203sp1.series | 6 +- + 4 files changed, 1088 insertions(+), 3 deletions(-) + create mode 100644 ldiskfs/kernel_patches/patches/oe2203sp3/ext4-mballoc-extra-checks.patch + create mode 100644 ldiskfs/kernel_patches/patches/oe2203sp3/ext4-prealloc.patch + create mode 100644 ldiskfs/kernel_patches/patches/oe2203sp3/ext4-simple-blockalloc.patch + +diff --git a/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-mballoc-extra-checks.patch b/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-mballoc-extra-checks.patch +new file mode 100644 +index 0000000000..09f2a49586 +--- /dev/null ++++ b/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-mballoc-extra-checks.patch +@@ -0,0 +1,320 @@ ++From 3b61206118f2a731016a3a113b7c2333843dd436 Mon Sep 17 00:00:00 2001 ++From: Xinliang Liu ++Date: Mon, 27 Nov 2023 08:23:12 +0000 ++Subject: [PATCH] ext4: mballoc extra checks ++ ++Signed-off-by: Xinliang Liu ++--- ++ fs/ext4/ext4.h | 1 + ++ fs/ext4/mballoc.c | 105 ++++++++++++++++++++++++++++++++++++++++------ ++ fs/ext4/mballoc.h | 2 +- ++ 3 files changed, 94 insertions(+), 14 deletions(-) ++ ++diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h ++index 26180ca..fd8d3a3 100644 ++--- a/fs/ext4/ext4.h +++++ b/fs/ext4/ext4.h ++@@ -3343,6 +3343,7 @@ struct ext4_group_info { ++ ext4_grpblk_t bb_fragments; /* nr of freespace fragments */ ++ ext4_grpblk_t bb_largest_free_order;/* order of largest frag in BG */ ++ struct list_head bb_prealloc_list; +++ unsigned long bb_prealloc_nr; ++ #ifdef DOUBLE_CHECK ++ void *bb_bitmap; ++ #endif ++diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c ++index a7b5eb7..4f40602 100644 ++--- a/fs/ext4/mballoc.c +++++ b/fs/ext4/mballoc.c ++@@ -345,7 +345,7 @@ static const char * const ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = { ++ "ext4_groupinfo_64k", "ext4_groupinfo_128k" ++ }; ++ ++-static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, +++static int ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, ++ ext4_group_t group); ++ static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, ++ ext4_group_t group); ++@@ -768,7 +768,7 @@ mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp) ++ } ++ ++ static noinline_for_stack ++-void ext4_mb_generate_buddy(struct super_block *sb, +++int ext4_mb_generate_buddy(struct super_block *sb, ++ void *buddy, void *bitmap, ext4_group_t group, ++ struct ext4_group_info *grp) ++ { ++@@ -812,6 +812,7 @@ void ext4_mb_generate_buddy(struct super_block *sb, ++ grp->bb_free = free; ++ ext4_mark_group_bitmap_corrupted(sb, group, ++ EXT4_GROUP_INFO_BBITMAP_CORRUPT); +++ return -EIO; ++ } ++ mb_set_largest_free_order(sb, grp); ++ ++@@ -820,6 +821,8 @@ void ext4_mb_generate_buddy(struct super_block *sb, ++ period = get_cycles() - period; ++ atomic_inc(&sbi->s_mb_buddies_generated); ++ atomic64_add(period, &sbi->s_mb_generation_time); +++ +++ return 0; ++ } ++ ++ /* The buddy information is attached the buddy cache inode ++@@ -924,7 +927,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) ++ } ++ ++ first_block = page->index * blocks_per_page; ++- for (i = 0; i < blocks_per_page; i++) { +++ for (i = 0; i < blocks_per_page && err == 0; i++) { ++ group = (first_block + i) >> 1; ++ if (group >= ngroups) ++ break; ++@@ -972,7 +975,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) ++ ext4_lock_group(sb, group); ++ /* init the buddy */ ++ memset(data, 0xff, blocksize); ++- ext4_mb_generate_buddy(sb, data, incore, group, grinfo); +++ err = ext4_mb_generate_buddy(sb, data, incore, group, grinfo); ++ ext4_unlock_group(sb, group); ++ incore = NULL; ++ } else { ++@@ -987,7 +990,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) ++ memcpy(data, bitmap, blocksize); ++ ++ /* mark all preallocated blks used in in-core bitmap */ ++- ext4_mb_generate_from_pa(sb, data, group); +++ err = ext4_mb_generate_from_pa(sb, data, group); ++ ext4_mb_generate_from_freelist(sb, data, group); ++ ext4_unlock_group(sb, group); ++ ++@@ -997,7 +1000,8 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) ++ incore = data; ++ } ++ } ++- SetPageUptodate(page); +++ if (likely(err == 0)) +++ SetPageUptodate(page); ++ ++ out: ++ if (bh) { ++@@ -2519,9 +2523,11 @@ static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) ++ static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) ++ { ++ struct super_block *sb = PDE_DATA(file_inode(seq->file)); +++ struct ext4_group_desc *gdp; ++ ext4_group_t group = (ext4_group_t) ((unsigned long) v); ++ int i; ++ int err, buddy_loaded = 0; +++ int free = 0; ++ struct ext4_buddy e4b; ++ struct ext4_group_info *grinfo; ++ unsigned char blocksize_bits = min_t(unsigned char, ++@@ -2534,7 +2540,7 @@ static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) ++ ++ group--; ++ if (group == 0) ++- seq_puts(seq, "#group: free frags first [" +++ seq_puts(seq, "#group: bfree gfree frags first pa [" ++ " 2^0 2^1 2^2 2^3 2^4 2^5 2^6 " ++ " 2^7 2^8 2^9 2^10 2^11 2^12 2^13 ]\n"); ++ ++@@ -2554,13 +2560,19 @@ static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) ++ buddy_loaded = 1; ++ } ++ +++ gdp = ext4_get_group_desc(sb, group, NULL); +++ if (gdp != NULL) +++ free = ext4_free_group_clusters(sb, gdp); +++ ++ memcpy(&sg, grinfo, i); ++ ++ if (buddy_loaded) ++ ext4_mb_unload_buddy(&e4b); ++ ++- seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free, ++- sg.info.bb_fragments, sg.info.bb_first_free); +++ seq_printf(seq, "#%-5lu: %-5u %-5u %-5u %-5u %-5lu [", +++ (long unsigned int)group, sg.info.bb_free, free, +++ sg.info.bb_fragments, sg.info.bb_first_free, +++ sg.info.bb_prealloc_nr); ++ for (i = 0; i <= 13; i++) ++ seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ? ++ sg.info.bb_counters[i] : 0); ++@@ -4097,25 +4109,74 @@ static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, ++ return; ++ } ++ +++/* +++ * check free blocks in bitmap match free block in group descriptor +++ * do this before taking preallocated blocks into account to be able +++ * to detect on-disk corruptions. The group lock should be hold by the +++ * caller. +++ */ +++int ext4_mb_check_ondisk_bitmap(struct super_block *sb, void *bitmap, +++ struct ext4_group_desc *gdp, int group) +++{ +++ unsigned short max = EXT4_CLUSTERS_PER_GROUP(sb); +++ unsigned short i, first, free = 0; +++ unsigned short free_in_gdp = ext4_free_group_clusters(sb, gdp); +++ +++ if (free_in_gdp == 0 && gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) +++ return 0; +++ +++ i = mb_find_next_zero_bit(bitmap, max, 0); +++ +++ while (i < max) { +++ first = i; +++ i = mb_find_next_bit(bitmap, max, i); +++ if (i > max) +++ i = max; +++ free += i - first; +++ if (i < max) +++ i = mb_find_next_zero_bit(bitmap, max, i); +++ } +++ +++ if (free != free_in_gdp) { +++ ext4_error(sb, "on-disk bitmap for group %d" +++ "corrupted: %u blocks free in bitmap, %u - in gd\n", +++ group, free, free_in_gdp); +++ return -EIO; +++ } +++ return 0; +++} +++ ++ /* ++ * the function goes through all preallocation in this group and marks them ++ * used in in-core bitmap. buddy must be generated from this bitmap ++ * Need to be called with ext4 group lock held ++ */ ++ static noinline_for_stack ++-void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, +++int ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, ++ ext4_group_t group) ++ { ++ struct ext4_group_info *grp = ext4_get_group_info(sb, group); ++ struct ext4_prealloc_space *pa; +++ struct ext4_group_desc *gdp; ++ struct list_head *cur; ++ ext4_group_t groupnr; ++ ext4_grpblk_t start; ++ int preallocated = 0; +++ int skip = 0, count = 0; +++ int err; ++ int len; ++ ++ if (!grp) ++- return; +++ return -EIO; +++ +++ gdp = ext4_get_group_desc(sb, group, NULL); +++ if (gdp == NULL) +++ return -EIO; +++ +++ /* before applying preallocations, check bitmap consistency */ +++ err = ext4_mb_check_ondisk_bitmap(sb, bitmap, gdp, group); +++ if (err) +++ return err; ++ ++ /* all form of preallocation discards first load group, ++ * so the only competing code is preallocation use. ++@@ -4132,13 +4193,23 @@ void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, ++ &groupnr, &start); ++ len = pa->pa_len; ++ spin_unlock(&pa->pa_lock); ++- if (unlikely(len == 0)) +++ if (unlikely(len == 0)) { +++ skip++; ++ continue; +++ } ++ BUG_ON(groupnr != group); ++ ext4_set_bits(bitmap, start, len); ++ preallocated += len; +++ count++; +++ } +++ if (count + skip != grp->bb_prealloc_nr) { +++ ext4_error(sb, "lost preallocations: " +++ "count %d, bb_prealloc_nr %lu, skip %d\n", +++ count, grp->bb_prealloc_nr, skip); +++ return -EIO; ++ } ++ mb_debug(sb, "preallocated %d for group %u\n", preallocated, group); +++ return 0; ++ } ++ ++ static void ext4_mb_mark_pa_deleted(struct super_block *sb, ++@@ -4222,6 +4293,7 @@ static void ext4_mb_put_pa(struct ext4_allocation_context *ac, ++ */ ++ ext4_lock_group(sb, grp); ++ list_del(&pa->pa_group_list); +++ ext4_get_group_info(sb, grp)->bb_prealloc_nr--; ++ ext4_unlock_group(sb, grp); ++ ++ spin_lock(pa->pa_obj_lock); ++@@ -4327,6 +4399,7 @@ adjust_bex: ++ pa->pa_inode = ac->ac_inode; ++ ++ list_add(&pa->pa_group_list, &grp->bb_prealloc_list); +++ grp->bb_prealloc_nr++; ++ ++ spin_lock(pa->pa_obj_lock); ++ list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list); ++@@ -4384,6 +4457,7 @@ ext4_mb_new_group_pa(struct ext4_allocation_context *ac) ++ pa->pa_inode = NULL; ++ ++ list_add(&pa->pa_group_list, &grp->bb_prealloc_list); +++ grp->bb_prealloc_nr++; ++ ++ /* ++ * We will later add the new pa to the right bucket ++@@ -4554,6 +4628,8 @@ ext4_mb_discard_group_preallocations(struct super_block *sb, ++ ++ spin_unlock(&pa->pa_lock); ++ +++ BUG_ON(grp->bb_prealloc_nr == 0); +++ grp->bb_prealloc_nr--; ++ list_del(&pa->pa_group_list); ++ list_add(&pa->u.pa_tmp_list, &list); ++ } ++@@ -4681,7 +4757,7 @@ repeat: ++ if (err) { ++ ext4_error_err(sb, -err, "Error %d loading buddy information for %u", ++ err, group); ++- continue; +++ return; ++ } ++ ++ bitmap_bh = ext4_read_block_bitmap(sb, group); ++@@ -4694,6 +4770,8 @@ repeat: ++ } ++ ++ ext4_lock_group(sb, group); +++ BUG_ON(e4b.bd_info->bb_prealloc_nr == 0); +++ e4b.bd_info->bb_prealloc_nr--; ++ list_del(&pa->pa_group_list); ++ ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); ++ ext4_unlock_group(sb, group); ++@@ -4991,6 +5069,7 @@ ext4_mb_discard_lg_preallocations(struct super_block *sb, ++ } ++ ext4_lock_group(sb, group); ++ list_del(&pa->pa_group_list); +++ ext4_get_group_info(sb, group)->bb_prealloc_nr--; ++ ext4_mb_release_group_pa(&e4b, pa); ++ ext4_unlock_group(sb, group); ++ ++diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h ++index 7be6288..448b2e8 100644 ++--- a/fs/ext4/mballoc.h +++++ b/fs/ext4/mballoc.h ++@@ -66,7 +66,7 @@ ++ /* ++ * for which requests use 2^N search using buddies ++ */ ++-#define MB_DEFAULT_ORDER2_REQS 2 +++#define MB_DEFAULT_ORDER2_REQS 8 ++ ++ /* ++ * default group prealloc size 512 blocks ++-- ++2.33.0 ++ +diff --git a/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-prealloc.patch b/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-prealloc.patch +new file mode 100644 +index 0000000000..bc64fdb084 +--- /dev/null ++++ b/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-prealloc.patch +@@ -0,0 +1,404 @@ ++From e64e4f631ac4aaa7fbaa97bcae213d32cb8fd6e3 Mon Sep 17 00:00:00 2001 ++From: Xinliang Liu ++Date: Mon, 27 Nov 2023 03:23:32 +0000 ++Subject: [PATCH] ext4: prealloc ++ ++Signed-off-by: Xinliang Liu ++--- ++ fs/ext4/ext4.h | 7 +- ++ fs/ext4/inode.c | 3 + ++ fs/ext4/mballoc.c | 219 +++++++++++++++++++++++++++++++++++----------- ++ fs/ext4/sysfs.c | 8 +- ++ 4 files changed, 182 insertions(+), 55 deletions(-) ++ ++diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h ++index 9e7dfc4..8a7df09 100644 ++--- a/fs/ext4/ext4.h +++++ b/fs/ext4/ext4.h ++@@ -1299,6 +1299,8 @@ extern void ext4_set_bits(void *bm, int cur, int len); ++ /* Metadata checksum algorithm codes */ ++ #define EXT4_CRC32C_CHKSUM 1 ++ +++#define EXT4_MAX_PREALLOC_TABLE 64 +++ ++ /* ++ * Structure of the super block ++ */ ++@@ -1534,11 +1536,13 @@ struct ext4_sb_info { ++ ++ /* tunables */ ++ unsigned long s_stripe; ++- unsigned int s_mb_stream_request; +++ unsigned long s_mb_small_req; +++ unsigned long s_mb_large_req; ++ unsigned int s_mb_max_to_scan; ++ unsigned int s_mb_min_to_scan; ++ unsigned int s_mb_stats; ++ unsigned int s_mb_order2_reqs; +++ unsigned long *s_mb_prealloc_table; ++ unsigned int s_mb_group_prealloc; ++ unsigned int s_mb_max_inode_prealloc; ++ unsigned int s_max_dir_size_kb; ++@@ -2827,6 +2831,7 @@ int ext4_fc_record_regions(struct super_block *sb, int ino, ++ int len, int replay); ++ ++ /* mballoc.c */ +++extern const struct proc_ops ext4_seq_prealloc_table_fops; ++ extern const struct seq_operations ext4_mb_seq_groups_ops; ++ extern long ext4_mb_stats; ++ extern long ext4_mb_max_to_scan; ++diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c ++index b2aa0ed..bda409e 100644 ++--- a/fs/ext4/inode.c +++++ b/fs/ext4/inode.c ++@@ -2773,6 +2773,9 @@ static int ext4_writepages(struct address_space *mapping, ++ PAGE_SIZE >> inode->i_blkbits); ++ } ++ +++ if (wbc->nr_to_write < sbi->s_mb_small_req) +++ wbc->nr_to_write = sbi->s_mb_small_req; +++ ++ if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) ++ range_whole = 1; ++ ++diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c ++index 929fc42..a7b5eb7 100644 ++--- a/fs/ext4/mballoc.c +++++ b/fs/ext4/mballoc.c ++@@ -2641,6 +2641,99 @@ int ext4_seq_mb_stats_show(struct seq_file *seq, void *offset) ++ return 0; ++ } ++ +++static int ext4_mb_check_and_update_prealloc(struct ext4_sb_info *sbi, +++ char *str, size_t cnt, +++ int update) +++{ +++ unsigned long value; +++ unsigned long prev = 0; +++ char *cur; +++ char *next; +++ char *end; +++ int num = 0; +++ +++ cur = str; +++ end = str + cnt; +++ while (cur < end) { +++ while ((cur < end) && (*cur == ' ')) cur++; +++ value = simple_strtol(cur, &next, 0); +++ if (value == 0) +++ break; +++ if (cur == next) +++ return -EINVAL; +++ +++ cur = next; +++ +++ if (value > (sbi->s_blocks_per_group - 1 - 1 - sbi->s_itb_per_group)) +++ return -EINVAL; +++ +++ /* they should add values in order */ +++ if (value <= prev) +++ return -EINVAL; +++ +++ if (update) +++ sbi->s_mb_prealloc_table[num] = value; +++ +++ prev = value; +++ num++; +++ } +++ +++ if (num > EXT4_MAX_PREALLOC_TABLE - 1) +++ return -EOVERFLOW; +++ +++ if (update) +++ sbi->s_mb_prealloc_table[num] = 0; +++ +++ return 0; +++} +++ +++static ssize_t ext4_mb_prealloc_table_proc_write(struct file *file, +++ const char __user *buf, +++ size_t cnt, loff_t *pos) +++{ +++ struct ext4_sb_info *sbi = EXT4_SB(PDE_DATA(file_inode(file))); +++ char str[128]; +++ int rc; +++ +++ if (cnt >= sizeof(str)) +++ return -EINVAL; +++ if (copy_from_user(str, buf, cnt)) +++ return -EFAULT; +++ +++ rc = ext4_mb_check_and_update_prealloc(sbi, str, cnt, 0); +++ if (rc) +++ return rc; +++ +++ rc = ext4_mb_check_and_update_prealloc(sbi, str, cnt, 1); +++ return rc ? rc : cnt; +++} +++ +++static int mb_prealloc_table_seq_show(struct seq_file *m, void *v) +++{ +++ struct ext4_sb_info *sbi = EXT4_SB(m->private); +++ int i; +++ +++ for (i = 0; i < EXT4_MAX_PREALLOC_TABLE && +++ sbi->s_mb_prealloc_table[i] != 0; i++) +++ seq_printf(m, "%ld ", sbi->s_mb_prealloc_table[i]); +++ seq_printf(m, "\n"); +++ +++ return 0; +++} +++ +++static int mb_prealloc_table_seq_open(struct inode *inode, struct file *file) +++{ +++ return single_open(file, mb_prealloc_table_seq_show, PDE_DATA(inode)); +++} +++ +++const struct proc_ops ext4_seq_prealloc_table_fops = { +++ .proc_open = mb_prealloc_table_seq_open, +++ .proc_read = seq_read, +++ .proc_lseek = seq_lseek, +++ .proc_release = single_release, +++ .proc_write = ext4_mb_prealloc_table_proc_write, +++}; +++ ++ static struct kmem_cache *get_groupinfo_cache(int blocksize_bits) ++ { ++ int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; ++@@ -2903,7 +2996,7 @@ static int ext4_groupinfo_create_slab(size_t size) ++ int ext4_mb_init(struct super_block *sb) ++ { ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++- unsigned i, j; +++ unsigned i, j, k, l; ++ unsigned offset, offset_incr; ++ unsigned max; ++ int ret; ++@@ -2951,7 +3044,6 @@ int ext4_mb_init(struct super_block *sb) ++ sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; ++ sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; ++ sbi->s_mb_stats = MB_DEFAULT_STATS; ++- sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; ++ sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; ++ sbi->s_mb_max_inode_prealloc = MB_DEFAULT_MAX_INODE_PREALLOC; ++ /* ++@@ -2976,9 +3068,29 @@ int ext4_mb_init(struct super_block *sb) ++ * RAID stripe size so that preallocations don't fragment ++ * the stripes. ++ */ ++- if (sbi->s_stripe > 1) { ++- sbi->s_mb_group_prealloc = roundup( ++- sbi->s_mb_group_prealloc, sbi->s_stripe); +++ +++ /* Allocate table once */ +++ sbi->s_mb_prealloc_table = kzalloc( +++ EXT4_MAX_PREALLOC_TABLE * sizeof(unsigned long), GFP_NOFS); +++ if (sbi->s_mb_prealloc_table == NULL) { +++ ret = -ENOMEM; +++ goto out; +++ } +++ +++ if (sbi->s_stripe == 0) { +++ for (k = 0, l = 4; k <= 9; ++k, l *= 2) +++ sbi->s_mb_prealloc_table[k] = l; +++ +++ sbi->s_mb_small_req = 256; +++ sbi->s_mb_large_req = 1024; +++ sbi->s_mb_group_prealloc = 512; +++ } else { +++ for (k = 0, l = sbi->s_stripe; k <= 2; ++k, l *= 2) +++ sbi->s_mb_prealloc_table[k] = l; +++ +++ sbi->s_mb_small_req = sbi->s_stripe; +++ sbi->s_mb_large_req = sbi->s_stripe * 8; +++ sbi->s_mb_group_prealloc = sbi->s_stripe * 4; ++ } ++ ++ sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); ++@@ -3006,6 +3118,7 @@ out_free_locality_groups: ++ free_percpu(sbi->s_locality_groups); ++ sbi->s_locality_groups = NULL; ++ out: +++ kfree(sbi->s_mb_prealloc_table); ++ kfree(sbi->s_mb_offsets); ++ sbi->s_mb_offsets = NULL; ++ kfree(sbi->s_mb_maxs); ++@@ -3278,7 +3391,6 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, ++ int err, len; ++ ++ BUG_ON(ac->ac_status != AC_STATUS_FOUND); ++- BUG_ON(ac->ac_b_ex.fe_len <= 0); ++ ++ sb = ac->ac_sb; ++ sbi = EXT4_SB(sb); ++@@ -3513,13 +3625,14 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, ++ { ++ struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); ++ struct ext4_super_block *es = sbi->s_es; ++- int bsbits, max; +++ int bsbits, i, wind; ++ ext4_lblk_t end; ++- loff_t size, start_off; +++ loff_t size; ++ loff_t orig_size __maybe_unused; ++ ext4_lblk_t start; ++ struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); ++ struct ext4_prealloc_space *pa; +++ unsigned long value, last_non_zero; ++ ++ /* do normalize only data requests, metadata requests ++ do not need preallocation */ ++@@ -3548,51 +3661,46 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, ++ size = size << bsbits; ++ if (size < i_size_read(ac->ac_inode)) ++ size = i_size_read(ac->ac_inode); ++- orig_size = size; +++ size = (size + ac->ac_sb->s_blocksize - 1) >> bsbits; +++ +++ start = wind = 0; +++ value = last_non_zero = 0; ++ ++- /* max size of free chunks */ ++- max = 2 << bsbits; ++- ++-#define NRL_CHECK_SIZE(req, size, max, chunk_size) \ ++- (req <= (size) || max <= (chunk_size)) ++- ++- /* first, try to predict filesize */ ++- /* XXX: should this table be tunable? */ ++- start_off = 0; ++- if (size <= 16 * 1024) { ++- size = 16 * 1024; ++- } else if (size <= 32 * 1024) { ++- size = 32 * 1024; ++- } else if (size <= 64 * 1024) { ++- size = 64 * 1024; ++- } else if (size <= 128 * 1024) { ++- size = 128 * 1024; ++- } else if (size <= 256 * 1024) { ++- size = 256 * 1024; ++- } else if (size <= 512 * 1024) { ++- size = 512 * 1024; ++- } else if (size <= 1024 * 1024) { ++- size = 1024 * 1024; ++- } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) { ++- start_off = ((loff_t)ac->ac_o_ex.fe_logical >> ++- (21 - bsbits)) << 21; ++- size = 2 * 1024 * 1024; ++- } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) { ++- start_off = ((loff_t)ac->ac_o_ex.fe_logical >> ++- (22 - bsbits)) << 22; ++- size = 4 * 1024 * 1024; ++- } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, ++- (8<<20)>>bsbits, max, 8 * 1024)) { ++- start_off = ((loff_t)ac->ac_o_ex.fe_logical >> ++- (23 - bsbits)) << 23; ++- size = 8 * 1024 * 1024; +++ /* let's choose preallocation window depending on file size */ +++ for (i = 0; i < EXT4_MAX_PREALLOC_TABLE; i++) { +++ value = sbi->s_mb_prealloc_table[i]; +++ if (value == 0) +++ break; +++ else +++ last_non_zero = value; +++ +++ if (size <= value) { +++ wind = value; +++ break; +++ } +++ } +++ +++ if (wind == 0) { +++ if (last_non_zero != 0) { +++ __u64 tstart, tend; +++ /* file is quite large, we now preallocate with +++ * the biggest configured window with regart to +++ * logical offset */ +++ wind = last_non_zero; +++ tstart = ac->ac_o_ex.fe_logical; +++ do_div(tstart, wind); +++ start = tstart * wind; +++ tend = ac->ac_o_ex.fe_logical + ac->ac_o_ex.fe_len - 1; +++ do_div(tend, wind); +++ tend = tend * wind + wind; +++ size = tend - start; +++ } ++ } else { ++- start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; ++- size = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb), ++- ac->ac_o_ex.fe_len) << bsbits; +++ size = wind; ++ } ++- size = size >> bsbits; ++- start = start_off >> bsbits; +++ +++ +++ orig_size = size; ++ ++ /* ++ * For tiny groups (smaller than 8MB) the chosen allocation ++@@ -3698,7 +3806,6 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, ++ (unsigned long) ac->ac_o_ex.fe_logical); ++ BUG(); ++ } ++- BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); ++ ++ /* now prepare goal request */ ++ ++@@ -4735,11 +4842,19 @@ static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) ++ ++ /* don't use group allocation for large files */ ++ size = max(size, isize); ++- if (size > sbi->s_mb_stream_request) { +++ if ((ac->ac_o_ex.fe_len >= sbi->s_mb_small_req) || +++ (size >= sbi->s_mb_large_req)) { ++ ac->ac_flags |= EXT4_MB_STREAM_ALLOC; ++ return; ++ } ++ +++ /* +++ * request is so large that we don't care about +++ * streaming - it overweights any possible seek +++ */ +++ if (ac->ac_o_ex.fe_len >= sbi->s_mb_large_req) +++ return; +++ ++ BUG_ON(ac->ac_lg != NULL); ++ /* ++ * locality group prealloc space are per cpu. The reason for having ++diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c ++index b0bb4a9..74ec068 100644 ++--- a/fs/ext4/sysfs.c +++++ b/fs/ext4/sysfs.c ++@@ -218,7 +218,8 @@ EXT4_RW_ATTR_SBI_UI(mb_stats, s_mb_stats); ++ EXT4_RW_ATTR_SBI_UI(mb_max_to_scan, s_mb_max_to_scan); ++ EXT4_RW_ATTR_SBI_UI(mb_min_to_scan, s_mb_min_to_scan); ++ EXT4_RW_ATTR_SBI_UI(mb_order2_req, s_mb_order2_reqs); ++-EXT4_RW_ATTR_SBI_UI(mb_stream_req, s_mb_stream_request); +++EXT4_RW_ATTR_SBI_UI(mb_small_req, s_mb_small_req); +++EXT4_RW_ATTR_SBI_UI(mb_large_req, s_mb_large_req); ++ EXT4_RW_ATTR_SBI_UI(mb_group_prealloc, s_mb_group_prealloc); ++ EXT4_RW_ATTR_SBI_UI(mb_max_inode_prealloc, s_mb_max_inode_prealloc); ++ EXT4_RW_ATTR_SBI_UI(extent_max_zeroout_kb, s_extent_max_zeroout_kb); ++@@ -266,7 +267,8 @@ static struct attribute *ext4_attrs[] = { ++ ATTR_LIST(mb_max_to_scan), ++ ATTR_LIST(mb_min_to_scan), ++ ATTR_LIST(mb_order2_req), ++- ATTR_LIST(mb_stream_req), +++ ATTR_LIST(mb_small_req), +++ ATTR_LIST(mb_large_req), ++ ATTR_LIST(mb_group_prealloc), ++ ATTR_LIST(mb_max_inode_prealloc), ++ ATTR_LIST(max_writeback_mb_bump), ++@@ -541,6 +543,8 @@ int ext4_register_sysfs(struct super_block *sb) ++ &ext4_mb_seq_groups_ops, sb); ++ proc_create_single_data("mb_stats", 0444, sbi->s_proc, ++ ext4_seq_mb_stats_show, sb); +++ proc_create_data("prealloc_table", S_IRUGO, sbi->s_proc, +++ &ext4_seq_prealloc_table_fops, sb); ++ } ++ return 0; ++ } ++-- ++2.33.0 ++ +diff --git a/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-simple-blockalloc.patch b/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-simple-blockalloc.patch +new file mode 100644 +index 0000000000..a9a79c4c2e +--- /dev/null ++++ b/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-simple-blockalloc.patch +@@ -0,0 +1,361 @@ ++From 20d4652a1de56f5354552f704947097bcc4f8eb2 Mon Sep 17 00:00:00 2001 ++From: Xinliang Liu ++Date: Mon, 27 Nov 2023 09:24:43 +0000 ++Subject: [PATCH] ext4: simple blockalloc ++ ++Signed-off-by: Xinliang Liu ++--- ++ fs/ext4/ext4.h | 5 ++ ++ fs/ext4/mballoc.c | 136 ++++++++++++++++++++++++++++++++++++++++++++++ ++ fs/ext4/mballoc.h | 3 + ++ fs/ext4/sysfs.c | 52 ++++++++++++++++++ ++ 4 files changed, 196 insertions(+) ++ ++diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h ++index fe8fe65..0c4f853 100644 ++--- a/fs/ext4/ext4.h +++++ b/fs/ext4/ext4.h ++@@ -1561,6 +1561,9 @@ struct ext4_sb_info { ++ unsigned int s_mb_min_to_scan; ++ unsigned int s_mb_stats; ++ unsigned int s_mb_order2_reqs; +++ ext4_fsblk_t s_mb_c1_blocks; +++ ext4_fsblk_t s_mb_c2_blocks; +++ ext4_fsblk_t s_mb_c3_blocks; ++ unsigned long *s_mb_prealloc_table; ++ unsigned int s_mb_group_prealloc; ++ unsigned int s_mb_max_inode_prealloc; ++@@ -1584,6 +1587,7 @@ struct ext4_sb_info { ++ atomic64_t s_bal_cX_groups_considered[4]; ++ atomic64_t s_bal_cX_hits[4]; ++ atomic64_t s_bal_cX_failed[4]; /* cX loop didn't find blocks */ +++ atomic64_t s_bal_cX_skipped[4]; ++ atomic_t s_mb_buddies_generated; /* number of buddies generated */ ++ atomic64_t s_mb_generation_time; ++ atomic_t s_mb_lost_chunks; ++@@ -3003,6 +3007,7 @@ int ext4_fc_record_regions(struct super_block *sb, int ino, ++ /* mballoc.c */ ++ extern const struct proc_ops ext4_seq_prealloc_table_fops; ++ extern const struct seq_operations ext4_mb_seq_groups_ops; +++extern const struct proc_ops ext4_mb_seq_alloc_fops; ++ extern const struct proc_ops ext4_seq_mb_last_group_fops; ++ extern int ext4_mb_seq_last_start_seq_show(struct seq_file *m, void *v); ++ extern long ext4_mb_stats; ++diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c ++index bd3938c..967a902 100644 ++--- a/fs/ext4/mballoc.c +++++ b/fs/ext4/mballoc.c ++@@ -2303,6 +2303,20 @@ void ext4_mb_prefetch_fini(struct super_block *sb, ext4_group_t group, ++ } ++ } ++ +++static u64 available_blocks_count(struct ext4_sb_info *sbi) +++{ +++ ext4_fsblk_t resv_blocks; +++ u64 bfree; +++ struct ext4_super_block *es = sbi->s_es; +++ +++ resv_blocks = EXT4_C2B(sbi, atomic64_read(&sbi->s_resv_clusters)); +++ bfree = percpu_counter_sum_positive(&sbi->s_freeclusters_counter) - +++ percpu_counter_sum_positive(&sbi->s_dirtyclusters_counter); +++ +++ bfree = EXT4_C2B(sbi, max_t(s64, bfree, 0)); +++ return bfree - (ext4_r_blocks_count(es) + resv_blocks); +++} +++ ++ static noinline_for_stack int ++ ext4_mb_regular_allocator(struct ext4_allocation_context *ac) ++ { ++@@ -2313,6 +2327,7 @@ ext4_mb_regular_allocator(struct ext4_allocation_context *ac) ++ struct ext4_sb_info *sbi; ++ struct super_block *sb; ++ struct ext4_buddy e4b; +++ ext4_fsblk_t avail_blocks; ++ int lost; ++ ++ sb = ac->ac_sb; ++@@ -2366,6 +2381,21 @@ ext4_mb_regular_allocator(struct ext4_allocation_context *ac) ++ ++ /* Let's just scan groups to find more-less suitable blocks */ ++ cr = ac->ac_2order ? 0 : 1; +++ +++ /* Choose what loop to pass based on disk fullness */ +++ avail_blocks = available_blocks_count(sbi) ; +++ +++ if (avail_blocks < sbi->s_mb_c3_blocks) { +++ cr = 3; +++ atomic64_inc(&sbi->s_bal_cX_skipped[2]); +++ } else if(avail_blocks < sbi->s_mb_c2_blocks) { +++ cr = 2; +++ atomic64_inc(&sbi->s_bal_cX_skipped[1]); +++ } else if(avail_blocks < sbi->s_mb_c1_blocks) { +++ cr = 1; +++ atomic64_inc(&sbi->s_bal_cX_skipped[0]); +++ } +++ ++ /* ++ * cr == 0 try to get exact allocation, ++ * cr == 3 try to get anything ++@@ -2484,6 +2514,9 @@ repeat: ++ cr = 3; ++ goto repeat; ++ } +++ /* Processed all groups and haven't found blocks */ +++ if (i == ngroups) +++ atomic64_inc(&sbi->s_bal_cX_failed[cr]); ++ } ++ ++ if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND) ++@@ -2810,6 +2843,95 @@ const struct proc_ops ext4_seq_mb_last_group_fops = { ++ .proc_write = ext4_mb_last_group_write, ++ }; ++ +++static int mb_seq_alloc_show(struct seq_file *seq, void *v) +++{ +++ struct super_block *sb = seq->private; +++ struct ext4_sb_info *sbi = EXT4_SB(sb); +++ +++ seq_printf(seq, "mballoc:\n"); +++ seq_printf(seq, "\tblocks: %u\n", atomic_read(&sbi->s_bal_allocated)); +++ seq_printf(seq, "\treqs: %u\n", atomic_read(&sbi->s_bal_reqs)); +++ seq_printf(seq, "\tsuccess: %u\n", atomic_read(&sbi->s_bal_success)); +++ +++ seq_printf(seq, "\textents_scanned: %u\n", +++ atomic_read(&sbi->s_bal_ex_scanned)); +++ seq_printf(seq, "\t\tgoal_hits: %u\n", atomic_read(&sbi->s_bal_goals)); +++ seq_printf(seq, "\t\t2^n_hits: %u\n", atomic_read(&sbi->s_bal_2orders)); +++ seq_printf(seq, "\t\tbreaks: %u\n", atomic_read(&sbi->s_bal_breaks)); +++ seq_printf(seq, "\t\tlost: %u\n", atomic_read(&sbi->s_mb_lost_chunks)); +++ +++ seq_printf(seq, "\tuseless_c1_loops: %llu\n", +++ atomic64_read(&sbi->s_bal_cX_failed[0])); +++ seq_printf(seq, "\tuseless_c2_loops: %llu\n", +++ atomic64_read(&sbi->s_bal_cX_failed[1])); +++ seq_printf(seq, "\tuseless_c3_loops: %llu\n", +++ atomic64_read(&sbi->s_bal_cX_failed[2])); +++ seq_printf(seq, "\tskipped_c1_loops: %llu\n", +++ atomic64_read(&sbi->s_bal_cX_skipped[0])); +++ seq_printf(seq, "\tskipped_c2_loops: %llu\n", +++ atomic64_read(&sbi->s_bal_cX_skipped[1])); +++ seq_printf(seq, "\tskipped_c3_loops: %llu\n", +++ atomic64_read(&sbi->s_bal_cX_skipped[2])); +++ seq_printf(seq, "\tbuddies_generated: %u\n", +++ atomic_read(&sbi->s_mb_buddies_generated)); +++ seq_printf(seq, "\tbuddies_time_used: %llu\n", +++ atomic64_read(&sbi->s_mb_generation_time)); +++ seq_printf(seq, "\tpreallocated: %u\n", +++ atomic_read(&sbi->s_mb_preallocated)); +++ seq_printf(seq, "\tdiscarded: %u\n", +++ atomic_read(&sbi->s_mb_discarded)); +++ return 0; +++} +++ +++static ssize_t mb_seq_alloc_write(struct file *file, +++ const char __user *buf, +++ size_t cnt, loff_t *pos) +++{ +++ struct ext4_sb_info *sbi = EXT4_SB(PDE_DATA(file_inode(file))); +++ +++ atomic_set(&sbi->s_bal_allocated, 0), +++ atomic_set(&sbi->s_bal_reqs, 0), +++ atomic_set(&sbi->s_bal_success, 0); +++ +++ atomic_set(&sbi->s_bal_ex_scanned, 0), +++ atomic_set(&sbi->s_bal_goals, 0), +++ atomic_set(&sbi->s_bal_2orders, 0), +++ atomic_set(&sbi->s_bal_breaks, 0), +++ atomic_set(&sbi->s_mb_lost_chunks, 0); +++ +++ atomic64_set(&sbi->s_bal_cX_failed[0], 0), +++ atomic64_set(&sbi->s_bal_cX_failed[1], 0), +++ atomic64_set(&sbi->s_bal_cX_failed[2], 0); +++ atomic64_set(&sbi->s_bal_cX_failed[3], 0); +++ +++ atomic64_set(&sbi->s_bal_cX_skipped[0], 0), +++ atomic64_set(&sbi->s_bal_cX_skipped[1], 0), +++ atomic64_set(&sbi->s_bal_cX_skipped[2], 0); +++ atomic64_set(&sbi->s_bal_cX_skipped[3], 0); +++ +++ +++ atomic_set(&sbi->s_mb_buddies_generated, 0); +++ atomic64_set(&sbi->s_mb_generation_time, 0); +++ +++ atomic_set(&sbi->s_mb_preallocated, 0), +++ atomic_set(&sbi->s_mb_discarded, 0); +++ +++ return cnt; +++} +++ +++static int mb_seq_alloc_open(struct inode *inode, struct file *file) +++{ +++ return single_open(file, mb_seq_alloc_show, PDE_DATA(inode)); +++} +++ +++const struct proc_ops ext4_mb_seq_alloc_fops = { +++ .proc_open = mb_seq_alloc_open, +++ .proc_read = seq_read, +++ .proc_lseek = seq_lseek, +++ .proc_release = single_release, +++ .proc_write = mb_seq_alloc_write, +++}; +++ ++ int ext4_mb_seq_last_start_seq_show(struct seq_file *m, void *v) ++ { ++ struct ext4_sb_info *sbi = EXT4_SB(m->private); ++@@ -3068,6 +3190,7 @@ static int ext4_groupinfo_create_slab(size_t size) ++ return 0; ++ } ++ +++#define THRESHOLD_BLOCKS(ts) (ext4_blocks_count(sbi->s_es) / 100 * ts) ++ int ext4_mb_init(struct super_block *sb) ++ { ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++@@ -3121,6 +3244,9 @@ int ext4_mb_init(struct super_block *sb) ++ sbi->s_mb_stats = MB_DEFAULT_STATS; ++ sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; ++ sbi->s_mb_max_inode_prealloc = MB_DEFAULT_MAX_INODE_PREALLOC; +++ sbi->s_mb_c1_blocks = THRESHOLD_BLOCKS(MB_DEFAULT_C1_THRESHOLD); +++ sbi->s_mb_c2_blocks = THRESHOLD_BLOCKS(MB_DEFAULT_C2_THRESHOLD); +++ sbi->s_mb_c3_blocks = THRESHOLD_BLOCKS(MB_DEFAULT_C3_THRESHOLD); ++ /* ++ * The default group preallocation is 512, which for 4k block ++ * sizes translates to 2 megabytes. However for bigalloc file ++@@ -3261,6 +3387,16 @@ int ext4_mb_release(struct super_block *sb) ++ atomic_read(&sbi->s_bal_allocated), ++ atomic_read(&sbi->s_bal_reqs), ++ atomic_read(&sbi->s_bal_success)); +++ ext4_msg(sb, KERN_INFO, +++ "mballoc: (%llu, %llu, %llu) useless c(0,1,2) loops", +++ atomic64_read(&sbi->s_bal_cX_failed[0]), +++ atomic64_read(&sbi->s_bal_cX_failed[1]), +++ atomic64_read(&sbi->s_bal_cX_failed[2])); +++ ext4_msg(sb, KERN_INFO, +++ "mballoc: (%llu, %llu, %llu) skipped c(0,1,2) loops", +++ atomic64_read(&sbi->s_bal_cX_skipped[0]), +++ atomic64_read(&sbi->s_bal_cX_skipped[1]), +++ atomic64_read(&sbi->s_bal_cX_skipped[2])); ++ ext4_msg(sb, KERN_INFO, ++ "mballoc: %u extents scanned, %u groups scanned, %u goal hits, " ++ "%u 2^N hits, %u breaks, %u lost", ++diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h ++index 90d6772..ea9a956 100644 ++--- a/fs/ext4/mballoc.h +++++ b/fs/ext4/mballoc.h ++@@ -68,6 +68,9 @@ ++ * for which requests use 2^N search using buddies ++ */ ++ #define MB_DEFAULT_ORDER2_REQS 8 +++#define MB_DEFAULT_C1_THRESHOLD 25 +++#define MB_DEFAULT_C2_THRESHOLD 15 +++#define MB_DEFAULT_C3_THRESHOLD 5 ++ ++ /* ++ * default group prealloc size 512 blocks ++diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c ++index 93d3ff5..91be249 100644 ++--- a/fs/ext4/sysfs.c +++++ b/fs/ext4/sysfs.c ++@@ -21,6 +21,9 @@ ++ typedef enum { ++ attr_noop, ++ attr_delayed_allocation_blocks, +++ attr_mb_c1_threshold, +++ attr_mb_c2_threshold, +++ attr_mb_c3_threshold, ++ attr_session_write_kbytes, ++ attr_lifetime_write_kbytes, ++ attr_reserved_clusters, ++@@ -141,6 +144,32 @@ static ssize_t journal_task_show(struct ext4_sb_info *sbi, char *buf) ++ task_pid_vnr(sbi->s_journal->j_task)); ++ } ++ +++#define THRESHOLD_PERCENT(ts) (ts * 100 / ext4_blocks_count(sbi->s_es)) +++ +++static int save_threshold_percent(struct ext4_sb_info *sbi, const char *buf, +++ ext4_fsblk_t *blocks) +++{ +++ unsigned long long val; +++ +++ int ret; +++ +++ ret = kstrtoull(skip_spaces(buf), 0, &val); +++ if (ret || val > 100) +++ return -EINVAL; +++ +++ *blocks = val * ext4_blocks_count(sbi->s_es) / 100; +++ return 0; +++} +++ +++static ssize_t mb_threshold_store(struct ext4_sb_info *sbi, +++ const char *buf, size_t count, +++ ext4_fsblk_t *blocks) +++{ +++ int ret = save_threshold_percent(sbi, buf, blocks); +++ +++ return ret ?: count; +++} +++ ++ #define EXT4_ATTR(_name,_mode,_id) \ ++ static struct ext4_attr ext4_attr_##_name = { \ ++ .attr = {.name = __stringify(_name), .mode = _mode }, \ ++@@ -210,6 +239,9 @@ EXT4_ATTR_FUNC(session_write_kbytes, 0444); ++ EXT4_ATTR_FUNC(lifetime_write_kbytes, 0444); ++ EXT4_ATTR_FUNC(reserved_clusters, 0644); ++ EXT4_ATTR_FUNC(sra_exceeded_retry_limit, 0444); +++EXT4_ATTR_FUNC(mb_c1_threshold, 0644); +++EXT4_ATTR_FUNC(mb_c2_threshold, 0644); +++EXT4_ATTR_FUNC(mb_c3_threshold, 0644); ++ ++ EXT4_ATTR_OFFSET(inode_readahead_blks, 0644, inode_readahead, ++ ext4_sb_info, s_inode_readahead_blks); ++@@ -264,6 +296,9 @@ static struct attribute *ext4_attrs[] = { ++ ATTR_LIST(lifetime_write_kbytes), ++ ATTR_LIST(reserved_clusters), ++ ATTR_LIST(sra_exceeded_retry_limit), +++ ATTR_LIST(mb_c1_threshold), +++ ATTR_LIST(mb_c2_threshold), +++ ATTR_LIST(mb_c3_threshold), ++ ATTR_LIST(inode_readahead_blks), ++ ATTR_LIST(inode_goal), ++ ATTR_LIST(max_dir_size), ++@@ -383,6 +418,15 @@ static ssize_t ext4_attr_show(struct kobject *kobj, ++ return snprintf(buf, PAGE_SIZE, "%llu\n", ++ (s64) EXT4_C2B(sbi, ++ percpu_counter_sum(&sbi->s_dirtyclusters_counter))); +++ case attr_mb_c1_threshold: +++ return scnprintf(buf, PAGE_SIZE, "%llu\n", +++ THRESHOLD_PERCENT(sbi->s_mb_c1_blocks)); +++ case attr_mb_c2_threshold: +++ return scnprintf(buf, PAGE_SIZE, "%llu\n", +++ THRESHOLD_PERCENT(sbi->s_mb_c2_blocks)); +++ case attr_mb_c3_threshold: +++ return scnprintf(buf, PAGE_SIZE, "%llu\n", +++ THRESHOLD_PERCENT(sbi->s_mb_c3_blocks)); ++ case attr_session_write_kbytes: ++ return session_write_kbytes_show(sbi, buf); ++ case attr_lifetime_write_kbytes: ++@@ -488,6 +532,12 @@ static ssize_t ext4_attr_store(struct kobject *kobj, ++ return inode_readahead_blks_store(sbi, buf, len); ++ case attr_trigger_test_error: ++ return trigger_test_error(sbi, buf, len); +++ case attr_mb_c1_threshold: +++ return mb_threshold_store(sbi, buf, len, &sbi->s_mb_c1_blocks); +++ case attr_mb_c2_threshold: +++ return mb_threshold_store(sbi, buf, len, &sbi->s_mb_c2_blocks); +++ case attr_mb_c3_threshold: +++ return mb_threshold_store(sbi, buf, len, &sbi->s_mb_c3_blocks); ++ } ++ return 0; ++ } ++@@ -559,6 +609,8 @@ int ext4_register_sysfs(struct super_block *sb) ++ &ext4_seq_mb_last_group_fops, sb); ++ proc_create_single_data("mb_last_start", S_IRUGO, sbi->s_proc, ++ ext4_mb_seq_last_start_seq_show, sb); +++ proc_create_data("mb_alloc_stats", S_IFREG | S_IRUGO | S_IWUSR, +++ sbi->s_proc, &ext4_mb_seq_alloc_fops, sb); ++ } ++ return 0; ++ } ++-- ++2.33.0 ++ +diff --git a/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series b/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series +index 82711ec7a5..68f1048c83 100644 +--- a/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series ++++ b/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series +@@ -1,10 +1,10 @@ + oe2203/ext4-inode-version.patch + linux-5.4/ext4-lookup-dotdot.patch + suse15/ext4-print-inum-in-htree-warning.patch +-linux-5.8/ext4-prealloc.patch ++oe2203sp3/ext4-prealloc.patch + ubuntu18/ext4-osd-iop-common.patch + oe2203/ext4-misc.patch +-linux-5.8/ext4-mballoc-extra-checks.patch ++oe2203sp3/ext4-mballoc-extra-checks.patch + linux-5.4/ext4-hash-indexed-dir-dotdot-update.patch + linux-5.8/ext4-kill-dx-root.patch + linux-5.8/ext4-mballoc-pa-free-mismatch.patch +@@ -23,7 +23,7 @@ rhel7.6/ext4-export-orphan-add.patch + linux-5.8/ext4-export-mb-stream-allocator-variables.patch + ubuntu19/ext4-iget-with-flags.patch + linux-5.4/export-ext4fs-dirhash-helper.patch +-oe2203/ext4-simple-blockalloc.patch ++oe2203sp3/ext4-simple-blockalloc.patch + oe2203/ext4-xattr-disable-credits-check.patch + linux-5.8/ext4-no-max-dir-size-limit-for-iam-objects.patch + rhel8/ext4-ialloc-uid-gid-and-pass-owner-down.patch +-- +2.33.0 + diff --git a/0046-Fix-sanity-running-stuck.patch b/0046-Fix-sanity-running-stuck.patch new file mode 100644 index 0000000000000000000000000000000000000000..7e0f55d7a538a392a6227a4be8fa0aecbc5f2ee0 --- /dev/null +++ b/0046-Fix-sanity-running-stuck.patch @@ -0,0 +1,262 @@ +From 015df9103b41fc7ab3e4ad478cb9c8ee69f3268b Mon Sep 17 00:00:00 2001 +From: Xinliang Liu +Date: Mon, 18 Dec 2023 02:05:59 +0000 +Subject: [PATCH 46/49] Fix sanity running stuck + +Fix sanity test suite runing stuck, via reverting series patchset[1]. +See bug: https://gitee.com/src-openeuler/lustre/issues/I8OSQ4 + +[1] ext4: clean up ea_inode handling +https://lore.kernel.org/all/20230524034951.779531-1-tytso@mit.edu/#r + +Fixes: #I8OSQ4 +Signed-off-by: Xinliang Liu +--- + ...ries-ext4-clean-up-ea_inode-handling.patch | 225 ++++++++++++++++++ + .../series/ldiskfs-5.10.0-oe2203sp1.series | 1 + + 2 files changed, 226 insertions(+) + create mode 100644 ldiskfs/kernel_patches/patches/oe2203/Revert-series-ext4-clean-up-ea_inode-handling.patch + +diff --git a/ldiskfs/kernel_patches/patches/oe2203/Revert-series-ext4-clean-up-ea_inode-handling.patch b/ldiskfs/kernel_patches/patches/oe2203/Revert-series-ext4-clean-up-ea_inode-handling.patch +new file mode 100644 +index 0000000000..81dee6f59f +--- /dev/null ++++ b/ldiskfs/kernel_patches/patches/oe2203/Revert-series-ext4-clean-up-ea_inode-handling.patch +@@ -0,0 +1,225 @@ ++From 7bd3c4c17a9734e8b7ebed5495d5a4b4e1ee667d Mon Sep 17 00:00:00 2001 ++From: Xinliang Liu ++Date: Fri, 15 Dec 2023 08:39:35 +0000 ++Subject: [PATCH] Revert series "ext4: clean up ea_inode handling" ++ ++Revert series: ++https://lore.kernel.org/all/20230524034951.779531-1-tytso@mit.edu/ ++Becuase it seems ploblematic with ldiskfs and stucks the sanity test suite. ++See bug: https://gitee.com/src-openeuler/lustre/issues/I8OSQ4 ++ ++Revert "ext4: add lockdep annotations for i_data_sem for ea_inode's" ++ ++This reverts commit a08ff710284c1970f93c69593d50f8cf82e74518. ++ ++Signed-off-by: Xinliang Liu ++ ++Revert "ext4: disallow ea_inodes with extended attributes" ++ ++This reverts commit d1c9aab3dfe9db90715372106d4e6997e62ec66e. ++ ++Signed-off-by: Xinliang Liu ++ ++Revert "ext4: set lockdep subclass for the ea_inode in ext4_xattr_inode_cache_find()" ++ ++This reverts commit 2f477b813221855823702277cca30d0a7f22675d. ++ ++Signed-off-by: Xinliang Liu ++ ++Revert "ext4: add EA_INODE checking to ext4_iget()" ++ ++This reverts commit 2174f45d71dbc2f0cf9408e8a890b3dacd03399c. ++ ++Signed-off-by: Xinliang Liu ++--- ++ fs/ext4/ext4.h | 5 +---- ++ fs/ext4/inode.c | 34 +++++----------------------------- ++ fs/ext4/xattr.c | 41 +++++++++++++++++++++++++++++------------ ++ 3 files changed, 35 insertions(+), 45 deletions(-) ++ ++diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h ++index 25a6e1eb6811..210dcad8b82d 100644 ++--- a/fs/ext4/ext4.h +++++ b/fs/ext4/ext4.h ++@@ -992,13 +992,11 @@ do { \ ++ * where the second inode has larger inode number ++ * than the first ++ * I_DATA_SEM_QUOTA - Used for quota inodes only ++- * I_DATA_SEM_EA - Used for ea_inodes only ++ */ ++ enum { ++ I_DATA_SEM_NORMAL = 0, ++ I_DATA_SEM_OTHER, ++ I_DATA_SEM_QUOTA, ++- I_DATA_SEM_EA ++ }; ++ ++ ++@@ -2894,8 +2892,7 @@ typedef enum { ++ EXT4_IGET_NORMAL = 0, ++ EXT4_IGET_SPECIAL = 0x0001, /* OK to iget a system inode */ ++ EXT4_IGET_HANDLE = 0x0002, /* Inode # is from a handle */ ++- EXT4_IGET_BAD = 0x0004, /* Allow to iget a bad inode */ ++- EXT4_IGET_EA_INODE = 0x0008 /* Inode should contain an EA value */ +++ EXT4_IGET_BAD = 0x0004 /* Allow to iget a bad inode */ ++ } ext4_iget_flags; ++ ++ extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ++diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c ++index 0757f6a6455f..d45ea81a64dc 100644 ++--- a/fs/ext4/inode.c +++++ b/fs/ext4/inode.c ++@@ -4813,24 +4813,6 @@ static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val) ++ inode_set_iversion_queried(inode, val); ++ } ++ ++-static const char *check_igot_inode(struct inode *inode, ext4_iget_flags flags) ++- ++-{ ++- if (flags & EXT4_IGET_EA_INODE) { ++- if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) ++- return "missing EA_INODE flag"; ++- if (ext4_test_inode_state(inode, EXT4_STATE_XATTR) || ++- EXT4_I(inode)->i_file_acl) ++- return "ea_inode with extended attributes"; ++- } else { ++- if ((EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) ++- return "unexpected EA_INODE flag"; ++- } ++- if (is_bad_inode(inode) && !(flags & EXT4_IGET_BAD)) ++- return "unexpected bad inode w/o EXT4_IGET_BAD"; ++- return NULL; ++-} ++- ++ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ++ ext4_iget_flags flags, const char *function, ++ unsigned int line) ++@@ -4839,7 +4821,6 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ++ struct ext4_inode *raw_inode; ++ struct ext4_inode_info *ei; ++ struct inode *inode; ++- const char *err_str; ++ journal_t *journal = EXT4_SB(sb)->s_journal; ++ long ret; ++ loff_t size; ++@@ -4863,14 +4844,8 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ++ inode = iget_locked(sb, ino); ++ if (!inode) ++ return ERR_PTR(-ENOMEM); ++- if (!(inode->i_state & I_NEW)) { ++- if ((err_str = check_igot_inode(inode, flags)) != NULL) { ++- ext4_error_inode(inode, function, line, 0, err_str); ++- iput(inode); ++- return ERR_PTR(-EFSCORRUPTED); ++- } +++ if (!(inode->i_state & I_NEW)) ++ return inode; ++- } ++ ++ ei = EXT4_I(inode); ++ iloc.bh = NULL; ++@@ -5139,9 +5114,10 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ++ if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb)) ++ ext4_error_inode(inode, function, line, 0, ++ "casefold flag without casefold feature"); ++- if ((err_str = check_igot_inode(inode, flags)) != NULL) { ++- ext4_error_inode(inode, function, line, 0, err_str); ++- ret = -EFSCORRUPTED; +++ if (is_bad_inode(inode) && !(flags & EXT4_IGET_BAD)) { +++ ext4_error_inode(inode, function, line, 0, +++ "bad inode without EXT4_IGET_BAD flag"); +++ ret = -EUCLEAN; ++ goto bad_inode; ++ } ++ ++diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c ++index d223fb41ecec..d8a5c27b834b 100644 ++--- a/fs/ext4/xattr.c +++++ b/fs/ext4/xattr.c ++@@ -123,11 +123,7 @@ ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array, ++ #ifdef CONFIG_LOCKDEP ++ void ext4_xattr_inode_set_class(struct inode *ea_inode) ++ { ++- struct ext4_inode_info *ei = EXT4_I(ea_inode); ++- ++ lockdep_set_subclass(&ea_inode->i_rwsem, 1); ++- (void) ei; /* shut up clang warning if !CONFIG_LOCKDEP */ ++- lockdep_set_subclass(&ei->i_data_sem, I_DATA_SEM_EA); ++ } ++ #endif ++ ++@@ -401,7 +397,7 @@ static int ext4_xattr_inode_iget(struct inode *parent, unsigned long ea_ino, ++ return -EFSCORRUPTED; ++ } ++ ++- inode = ext4_iget(parent->i_sb, ea_ino, EXT4_IGET_EA_INODE); +++ inode = ext4_iget(parent->i_sb, ea_ino, EXT4_IGET_NORMAL); ++ if (IS_ERR(inode)) { ++ err = PTR_ERR(inode); ++ ext4_error(parent->i_sb, ++@@ -409,6 +405,23 @@ static int ext4_xattr_inode_iget(struct inode *parent, unsigned long ea_ino, ++ err); ++ return err; ++ } +++ +++ if (is_bad_inode(inode)) { +++ ext4_error(parent->i_sb, +++ "error while reading EA inode %lu is_bad_inode", +++ ea_ino); +++ err = -EIO; +++ goto error; +++ } +++ +++ if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) { +++ ext4_error(parent->i_sb, +++ "EA inode %lu does not have EXT4_EA_INODE_FL flag", +++ ea_ino); +++ err = -EINVAL; +++ goto error; +++ } +++ ++ ext4_xattr_inode_set_class(inode); ++ ++ /* ++@@ -429,6 +442,9 @@ static int ext4_xattr_inode_iget(struct inode *parent, unsigned long ea_ino, ++ ++ *ea_inode = inode; ++ return 0; +++error: +++ iput(inode); +++ return err; ++ } ++ ++ /* Remove entry from mbcache when EA inode is getting evicted */ ++@@ -1487,11 +1503,11 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, ++ ++ while (ce) { ++ ea_inode = ext4_iget(inode->i_sb, ce->e_value, ++- EXT4_IGET_EA_INODE); ++- if (IS_ERR(ea_inode)) ++- goto next_entry; ++- ext4_xattr_inode_set_class(ea_inode); ++- if (i_size_read(ea_inode) == value_len && +++ EXT4_IGET_NORMAL); +++ if (!IS_ERR(ea_inode) && +++ !is_bad_inode(ea_inode) && +++ (EXT4_I(ea_inode)->i_flags & EXT4_EA_INODE_FL) && +++ i_size_read(ea_inode) == value_len && ++ !ext4_xattr_inode_read(ea_inode, ea_data, value_len) && ++ !ext4_xattr_inode_verify_hashes(ea_inode, NULL, ea_data, ++ value_len) && ++@@ -1501,8 +1517,9 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, ++ kvfree(ea_data); ++ return ea_inode; ++ } ++- iput(ea_inode); ++- next_entry: +++ +++ if (!IS_ERR(ea_inode)) +++ iput(ea_inode); ++ ce = mb_cache_entry_find_next(ea_inode_cache, ce); ++ } ++ kvfree(ea_data); ++-- ++2.34.1 ++ +diff --git a/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series b/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series +index 68f1048c83..621f92dfd5 100644 +--- a/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series ++++ b/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series +@@ -1,3 +1,4 @@ ++oe2203/Revert-series-ext4-clean-up-ea_inode-handling.patch + oe2203/ext4-inode-version.patch + linux-5.4/ext4-lookup-dotdot.patch + suse15/ext4-print-inum-in-htree-warning.patch +-- +2.33.0 + diff --git a/0047-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP3.patch b/0047-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP3.patch new file mode 100644 index 0000000000000000000000000000000000000000..02d49d3c565c8aa6e54aff5b4ba60564f140b240 --- /dev/null +++ b/0047-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP3.patch @@ -0,0 +1,127 @@ +From 65b6189d002dbea54fca0231f85d847cda684bd3 Mon Sep 17 00:00:00 2001 +From: Xinliang Liu +Date: Tue, 23 Jan 2024 02:41:59 +0000 +Subject: [PATCH 47/49] LU-16976 ldiskfs: add support for openEuler 22.03 SP3 + +Add lbuild support for oe2203.sp3. +Update ldiskfs patches for new kernel. + +Signed-off-by: Xinliang Liu +--- + contrib/lbuild/funcs.sh | 1 + + ...ries-ext4-clean-up-ea_inode-handling.patch | 18 ++++++++-------- + lustre/autoconf/lustre-core.m4 | 1 + + .../targets/5.10-oe2203sp3.target.in | 21 +++++++++++++++++++ + 4 files changed, 32 insertions(+), 9 deletions(-) + create mode 100644 lustre/kernel_patches/targets/5.10-oe2203sp3.target.in + +diff --git a/contrib/lbuild/funcs.sh b/contrib/lbuild/funcs.sh +index 8cbe00b5a8..a76323095b 100644 +--- a/contrib/lbuild/funcs.sh ++++ b/contrib/lbuild/funcs.sh +@@ -227,6 +227,7 @@ autodetect_target() { + oe2203) target="5.10-oe2203";; + oe2203.sp1) target="5.10-oe2203sp1";; + oe2203.sp2) target="5.10-oe2203sp2";; ++ oe2203.sp3) target="5.10-oe2203sp3";; + *) fatal 1 "I don't know what distro $distro is.\nEither update autodetect_target() or use the --target argument.";; + esac + +diff --git a/ldiskfs/kernel_patches/patches/oe2203/Revert-series-ext4-clean-up-ea_inode-handling.patch b/ldiskfs/kernel_patches/patches/oe2203/Revert-series-ext4-clean-up-ea_inode-handling.patch +index 81dee6f59f..c13899ec22 100644 +--- a/ldiskfs/kernel_patches/patches/oe2203/Revert-series-ext4-clean-up-ea_inode-handling.patch ++++ b/ldiskfs/kernel_patches/patches/oe2203/Revert-series-ext4-clean-up-ea_inode-handling.patch +@@ -1,4 +1,4 @@ +-From 7bd3c4c17a9734e8b7ebed5495d5a4b4e1ee667d Mon Sep 17 00:00:00 2001 ++From bd0a990771a934482be48df02465c35917cbbba5 Mon Sep 17 00:00:00 2001 + From: Xinliang Liu + Date: Fri, 15 Dec 2023 08:39:35 +0000 + Subject: [PATCH] Revert series "ext4: clean up ea_inode handling" +@@ -66,10 +66,10 @@ index 25a6e1eb6811..210dcad8b82d 100644 + + extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, + diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c +-index 0757f6a6455f..d45ea81a64dc 100644 ++index 412af3c5fde4..29ac7c77096d 100644 + --- a/fs/ext4/inode.c + +++ b/fs/ext4/inode.c +-@@ -4813,24 +4813,6 @@ static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val) ++@@ -4814,24 +4814,6 @@ static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val) + inode_set_iversion_queried(inode, val); + } + +@@ -94,7 +94,7 @@ index 0757f6a6455f..d45ea81a64dc 100644 + struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, + ext4_iget_flags flags, const char *function, + unsigned int line) +-@@ -4839,7 +4821,6 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ++@@ -4840,7 +4822,6 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, + struct ext4_inode *raw_inode; + struct ext4_inode_info *ei; + struct inode *inode; +@@ -102,7 +102,7 @@ index 0757f6a6455f..d45ea81a64dc 100644 + journal_t *journal = EXT4_SB(sb)->s_journal; + long ret; + loff_t size; +-@@ -4863,14 +4844,8 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ++@@ -4864,14 +4845,8 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); +@@ -118,10 +118,10 @@ index 0757f6a6455f..d45ea81a64dc 100644 + + ei = EXT4_I(inode); + iloc.bh = NULL; +-@@ -5139,9 +5114,10 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, +- if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb)) +- ext4_error_inode(inode, function, line, 0, +- "casefold flag without casefold feature"); ++@@ -5143,9 +5118,10 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ++ ret = -EFSCORRUPTED; ++ goto bad_inode; ++ } + - if ((err_str = check_igot_inode(inode, flags)) != NULL) { + - ext4_error_inode(inode, function, line, 0, err_str); + - ret = -EFSCORRUPTED; +diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 +index 4bb87dadaa..7e50ec1b52 100644 +--- a/lustre/autoconf/lustre-core.m4 ++++ b/lustre/autoconf/lustre-core.m4 +@@ -4478,6 +4478,7 @@ lustre/kernel_patches/targets/4.19-oe2003sp3.target + lustre/kernel_patches/targets/5.10-oe2203.target + lustre/kernel_patches/targets/5.10-oe2203sp1.target + lustre/kernel_patches/targets/5.10-oe2203sp2.target ++lustre/kernel_patches/targets/5.10-oe2203sp3.target + lustre/ldlm/Makefile + lustre/fid/Makefile + lustre/fid/autoMakefile +diff --git a/lustre/kernel_patches/targets/5.10-oe2203sp3.target.in b/lustre/kernel_patches/targets/5.10-oe2203sp3.target.in +new file mode 100644 +index 0000000000..de88200fb4 +--- /dev/null ++++ b/lustre/kernel_patches/targets/5.10-oe2203sp3.target.in +@@ -0,0 +1,21 @@ ++lnxmaj="5.10.0" ++lnxrel="184.0.0.97.oe2203sp3" ++ ++KERNEL_SRPM=kernel-${lnxmaj}-${lnxrel}.src.rpm ++SERIES="" ++EXTRA_VERSION=${lnxrel}_lustre.@VERSION@ ++LUSTRE_VERSION=@VERSION@ ++ ++DEVEL_PATH_ARCH_DELIMETER="." ++OFED_VERSION=inkernel ++ ++#SMP_ARCHS="i686 x86_64 ia64 ppc64" ++# openEuler doesn't use smp specific kernels ++SMP_ARCHS="" ++ ++for cc in gcc ; do ++ if which $cc >/dev/null 2>/dev/null ; then ++ export CC=$cc ++ break ++ fi ++done +-- +2.33.0 + diff --git a/0048-ldiskfs-fix-build-for-kernel-5.10.0-186.0.0.99.oe220.patch b/0048-ldiskfs-fix-build-for-kernel-5.10.0-186.0.0.99.oe220.patch new file mode 100644 index 0000000000000000000000000000000000000000..2b9b734e1acf0ff180001cc796b20213a41c6730 --- /dev/null +++ b/0048-ldiskfs-fix-build-for-kernel-5.10.0-186.0.0.99.oe220.patch @@ -0,0 +1,126 @@ +From 70db9bf354bb6acd4cec2c200c9a84cb753ff601 Mon Sep 17 00:00:00 2001 +From: Xinliang Liu +Date: Thu, 22 Feb 2024 03:45:03 +0000 +Subject: [PATCH 48/49] ldiskfs: fix build for kernel + 5.10.0-186.0.0.99.oe2203sp3 + +Signed-off-by: Xinliang Liu +--- + .../patches/oe2203sp3/ext4-prealloc.patch | 31 +++++++++---------- + 1 file changed, 15 insertions(+), 16 deletions(-) + +diff --git a/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-prealloc.patch b/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-prealloc.patch +index bc64fdb084..0ded1a7549 100644 +--- a/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-prealloc.patch ++++ b/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-prealloc.patch +@@ -1,6 +1,6 @@ +-From e64e4f631ac4aaa7fbaa97bcae213d32cb8fd6e3 Mon Sep 17 00:00:00 2001 ++From 92e6d50db5af32862f796737d2531f691a07e6f4 Mon Sep 17 00:00:00 2001 + From: Xinliang Liu +-Date: Mon, 27 Nov 2023 03:23:32 +0000 ++Date: Thu, 22 Feb 2024 03:41:24 +0000 + Subject: [PATCH] ext4: prealloc + + Signed-off-by: Xinliang Liu +@@ -12,10 +12,10 @@ Signed-off-by: Xinliang Liu + 4 files changed, 182 insertions(+), 55 deletions(-) + + diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h +-index 9e7dfc4..8a7df09 100644 ++index 87bbd21..e6999b6 100644 + --- a/fs/ext4/ext4.h + +++ b/fs/ext4/ext4.h +-@@ -1299,6 +1299,8 @@ extern void ext4_set_bits(void *bm, int cur, int len); ++@@ -1304,6 +1304,8 @@ extern void ext4_set_bits(void *bm, int cur, int len); + /* Metadata checksum algorithm codes */ + #define EXT4_CRC32C_CHKSUM 1 + +@@ -24,7 +24,7 @@ index 9e7dfc4..8a7df09 100644 + /* + * Structure of the super block + */ +-@@ -1534,11 +1536,13 @@ struct ext4_sb_info { ++@@ -1539,11 +1541,13 @@ struct ext4_sb_info { + + /* tunables */ + unsigned long s_stripe; +@@ -39,7 +39,7 @@ index 9e7dfc4..8a7df09 100644 + unsigned int s_mb_group_prealloc; + unsigned int s_mb_max_inode_prealloc; + unsigned int s_max_dir_size_kb; +-@@ -2827,6 +2831,7 @@ int ext4_fc_record_regions(struct super_block *sb, int ino, ++@@ -2832,6 +2836,7 @@ int ext4_fc_record_regions(struct super_block *sb, int ino, + int len, int replay); + + /* mballoc.c */ +@@ -48,10 +48,10 @@ index 9e7dfc4..8a7df09 100644 + extern long ext4_mb_stats; + extern long ext4_mb_max_to_scan; + diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c +-index b2aa0ed..bda409e 100644 ++index f8c11bb..613a9be 100644 + --- a/fs/ext4/inode.c + +++ b/fs/ext4/inode.c +-@@ -2773,6 +2773,9 @@ static int ext4_writepages(struct address_space *mapping, ++@@ -2774,6 +2774,9 @@ static int ext4_writepages(struct address_space *mapping, + PAGE_SIZE >> inode->i_blkbits); + } + +@@ -62,7 +62,7 @@ index b2aa0ed..bda409e 100644 + range_whole = 1; + + diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c +-index 929fc42..a7b5eb7 100644 ++index bc25bdb..a965535 100644 + --- a/fs/ext4/mballoc.c + +++ b/fs/ext4/mballoc.c + @@ -2641,6 +2641,99 @@ int ext4_seq_mb_stats_show(struct seq_file *seq, void *offset) +@@ -231,15 +231,14 @@ index 929fc42..a7b5eb7 100644 + + sb = ac->ac_sb; + sbi = EXT4_SB(sb); +-@@ -3513,13 +3625,14 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, ++@@ -3513,12 +3625,13 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, + { + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + struct ext4_super_block *es = sbi->s_es; + - int bsbits, max; ++- loff_t size, start_off, end; + + int bsbits, i, wind; +- ext4_lblk_t end; +-- loff_t size, start_off; +-+ loff_t size; +++ loff_t size, end; + loff_t orig_size __maybe_unused; + ext4_lblk_t start; + struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); +@@ -248,7 +247,7 @@ index 929fc42..a7b5eb7 100644 + + /* do normalize only data requests, metadata requests + do not need preallocation */ +-@@ -3548,51 +3661,46 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, ++@@ -3547,51 +3660,46 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, + size = size << bsbits; + if (size < i_size_read(ac->ac_inode)) + size = i_size_read(ac->ac_inode); +@@ -337,7 +336,7 @@ index 929fc42..a7b5eb7 100644 + + /* + * For tiny groups (smaller than 8MB) the chosen allocation +-@@ -3698,7 +3806,6 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, ++@@ -3699,7 +3807,6 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, + (unsigned long) ac->ac_o_ex.fe_logical); + BUG(); + } +@@ -345,7 +344,7 @@ index 929fc42..a7b5eb7 100644 + + /* now prepare goal request */ + +-@@ -4735,11 +4842,19 @@ static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) ++@@ -4732,11 +4839,19 @@ static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) + + /* don't use group allocation for large files */ + size = max(size, isize); +-- +2.33.0 + diff --git a/0049-Fix-build-for-5.10.0-197.0.0.110.oe2203sp3.patch b/0049-Fix-build-for-5.10.0-197.0.0.110.oe2203sp3.patch new file mode 100644 index 0000000000000000000000000000000000000000..6515cb3df7761ee53aab0a5270eaf70dbb3fb839 --- /dev/null +++ b/0049-Fix-build-for-5.10.0-197.0.0.110.oe2203sp3.patch @@ -0,0 +1,325 @@ +From c1a0dbc56cf49a75e73f1ebf366ef1beb63711b5 Mon Sep 17 00:00:00 2001 +From: Xinliang Liu +Date: Mon, 29 Apr 2024 04:23:02 +0000 +Subject: [PATCH 49/49] Fix build for 5.10.0-197.0.0.110.oe2203sp3 + +Signed-off-by: Xinliang Liu +--- + ...inode-block-bitmaps-handling-patches.patch | 291 ++++++++++++++++++ + .../series/ldiskfs-5.10.0-oe2203sp1.series | 2 +- + 2 files changed, 292 insertions(+), 1 deletion(-) + create mode 100644 ldiskfs/kernel_patches/patches/oe2203sp3/ext4-corrupted-inode-block-bitmaps-handling-patches.patch + +diff --git a/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-corrupted-inode-block-bitmaps-handling-patches.patch b/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-corrupted-inode-block-bitmaps-handling-patches.patch +new file mode 100644 +index 0000000000..51b98291ce +--- /dev/null ++++ b/ldiskfs/kernel_patches/patches/oe2203sp3/ext4-corrupted-inode-block-bitmaps-handling-patches.patch +@@ -0,0 +1,291 @@ ++commit 2963f3d09eb3a0817f87386c0bd7be7ce086809d ++Author: Wang Shilong ++AuthorDate: Tue Sep 8 21:54:29 2015 +0800 ++LU-7114 ldiskfs: corrupted bitmaps handling patches ++ ++This patch backported following patches from upstream: ++ ++163a203ddb36c36d4a1c942aececda0cc8d06aa7 ++ext4: mark block group as corrupt on block bitmap error ++ ++87a39389be3e3b007d341be510a7e4a0542bdf05 ++ext4: mark block group as corrupt on inode bitmap error ++ ++bdfb6ff4a255dcebeb09a901250e13a97eff75af ++ext4: mark group corrupt on group descriptor checksum ++ ++Also use ext4_warning() instead of ext4_error() so that ++filesystem don't become RO in default, and together ++with these patches,FS wil still be usable even such ++bad things happen. ++ ++Signed-off-by: Wang Shilong ++Change-Id: Ib4075aba7df6f7f59e89a90475405080acd43dd0 ++Reviewed-on: http://review.whamcloud.com/16312 ++Reviewed-by: Andreas Dilger ++Reviewed-by: Yang Sheng ++ ++NOTE: Ported to linux 5.16 keeps the ext4_warning() updates. ++--- ++ fs/ext4/balloc.c | 10 ++++---- ++ fs/ext4/ialloc.c | 6 ++--- ++ fs/ext4/mballoc.c | 63 +++++++++++++++++++---------------------------- ++ 3 files changed, 33 insertions(+), 46 deletions(-) ++ ++diff --git a/fs/ext4/balloc.c b/fs/ext4/balloc.c ++index a0fb0c4..84c5c78 100644 ++--- a/fs/ext4/balloc.c +++++ b/fs/ext4/balloc.c ++@@ -387,7 +387,7 @@ static int ext4_validate_block_bitmap(struct super_block *sb, ++ desc, bh) || ++ ext4_simulate_fail(sb, EXT4_SIM_BBITMAP_CRC))) { ++ ext4_unlock_group(sb, block_group); ++- ext4_error(sb, "bg %u: bad block bitmap checksum", block_group); +++ ext4_warning(sb, "bg %u: bad block bitmap checksum", block_group); ++ ext4_mark_group_bitmap_corrupted(sb, block_group, ++ EXT4_GROUP_INFO_BBITMAP_CORRUPT); ++ return -EFSBADCRC; ++@@ -395,8 +395,8 @@ static int ext4_validate_block_bitmap(struct super_block *sb, ++ blk = ext4_valid_block_bitmap(sb, desc, block_group, bh); ++ if (unlikely(blk != 0)) { ++ ext4_unlock_group(sb, block_group); ++- ext4_error(sb, "bg %u: block %llu: invalid block bitmap", ++- block_group, blk); +++ ext4_warning(sb, "bg %u: block %llu: invalid block bitmap", +++ block_group, blk); ++ ext4_mark_group_bitmap_corrupted(sb, block_group, ++ EXT4_GROUP_INFO_BBITMAP_CORRUPT); ++ return -EFSCORRUPTED; ++@@ -479,8 +479,8 @@ ext4_read_block_bitmap_nowait(struct super_block *sb, ext4_group_t block_group, ++ ext4_unlock_group(sb, block_group); ++ unlock_buffer(bh); ++ if (err) { ++- ext4_error(sb, "Failed to init block bitmap for group " ++- "%u: %d", block_group, err); +++ ext4_warning(sb, "Failed to init block bitmap for group " +++ "%u: %d", block_group, err); ++ goto out; ++ } ++ goto verify; ++diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c ++index 4840190..f73d3f8 100644 ++--- a/fs/ext4/ialloc.c +++++ b/fs/ext4/ialloc.c ++@@ -102,8 +102,8 @@ static int ext4_validate_inode_bitmap(struct super_block *sb, ++ EXT4_INODES_PER_GROUP(sb) / 8) || ++ ext4_simulate_fail(sb, EXT4_SIM_IBITMAP_CRC)) { ++ ext4_unlock_group(sb, block_group); ++- ext4_error(sb, "Corrupt inode bitmap - block_group = %u, " ++- "inode_bitmap = %llu", block_group, blk); +++ ext4_warning(sb, "Corrupt inode bitmap - block_group = %u, " +++ "inode_bitmap = %llu", block_group, blk); ++ ext4_mark_group_bitmap_corrupted(sb, block_group, ++ EXT4_GROUP_INFO_IBITMAP_CORRUPT); ++ return -EFSBADCRC; ++@@ -353,7 +353,7 @@ out: ++ if (!fatal) ++ fatal = err; ++ } else { ++- ext4_error(sb, "bit already cleared for inode %lu", ino); +++ ext4_warning(sb, "bit already cleared for inode %lu", ino); ++ ext4_mark_group_bitmap_corrupted(sb, block_group, ++ EXT4_GROUP_INFO_IBITMAP_CORRUPT); ++ } ++diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c ++index cff1df6..0ee37f0 100644 ++--- a/fs/ext4/mballoc.c +++++ b/fs/ext4/mballoc.c ++@@ -1138,10 +1138,14 @@ int ext4_mb_generate_buddy(struct super_block *sb, ++ grp->bb_fragments = fragments; ++ ++ if (free != grp->bb_free) { ++- ext4_grp_locked_error(sb, group, 0, 0, ++- "block bitmap and bg descriptor " ++- "inconsistent: %u vs %u free clusters", ++- free, grp->bb_free); +++ struct ext4_group_desc *gdp; +++ gdp = ext4_get_group_desc(sb, group, NULL); +++ ext4_warning(sb, "group %lu: block bitmap and bg descriptor " +++ "inconsistent: %u vs %u free clusters " +++ "%u in gd, %lu pa's", +++ (long unsigned int)group, free, grp->bb_free, +++ ext4_free_group_clusters(sb, gdp), +++ grp->bb_prealloc_nr); ++ /* ++ * If we intend to continue, we consider group descriptor ++ * corrupt and update bb_free using bitmap value ++@@ -1485,7 +1489,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, ++ int block; ++ int pnum; ++ int poff; ++- struct page *page; +++ struct page *page = NULL; ++ int ret; ++ struct ext4_group_info *grp; ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++@@ -1511,7 +1515,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, ++ */ ++ ret = ext4_mb_init_group(sb, group, gfp); ++ if (ret) ++- return ret; +++ goto err; ++ } ++ ++ /* ++@@ -1611,6 +1615,7 @@ err: ++ put_page(e4b->bd_buddy_page); ++ e4b->bd_buddy = NULL; ++ e4b->bd_bitmap = NULL; +++ ext4_warning(sb, "Error loading buddy information for %u", group); ++ return ret; ++ } ++ ++@@ -4587,9 +4592,11 @@ int ext4_mb_check_ondisk_bitmap(struct super_block *sb, void *bitmap, ++ } ++ ++ if (free != free_in_gdp) { ++- ext4_error(sb, "on-disk bitmap for group %d" +++ ext4_warning(sb, "on-disk bitmap for group %d" ++ "corrupted: %u blocks free in bitmap, %u - in gd\n", ++ group, free, free_in_gdp); +++ ext4_mark_group_bitmap_corrupted(sb, group, +++ EXT4_GROUP_INFO_BBITMAP_CORRUPT); ++ return -EIO; ++ } ++ return 0; ++@@ -4955,16 +4962,8 @@ ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh, ++ /* "free < pa->pa_free" means we maybe double alloc the same blocks, ++ * otherwise maybe leave some free blocks unavailable, no need to BUG.*/ ++ if ((free > pa->pa_free && !pa->pa_error) || (free < pa->pa_free)) { ++- ext4_error(sb, "pa free mismatch: [pa %p] " ++- "[phy %lu] [logic %lu] [len %u] [free %u] " ++- "[error %u] [inode %d] [freed %u]", pa, ++- (unsigned long)pa->pa_pstart, ++- (unsigned long)pa->pa_lstart, ++- pa->pa_len, (unsigned)pa->pa_free, ++- (unsigned)pa->pa_error, pa->pa_inode->i_ino, ++- free); ++ ext4_grp_locked_error(sb, group, 0, 0, "free %u, pa_free %u", ++- free, pa->pa_free); +++ free, pa->pa_free); ++ /* ++ * pa is already deleted so we use the value obtained ++ * from the bitmap and continue. ++@@ -5024,16 +5023,11 @@ ext4_mb_discard_group_preallocations(struct super_block *sb, ++ bitmap_bh = ext4_read_block_bitmap(sb, group); ++ if (IS_ERR(bitmap_bh)) { ++ err = PTR_ERR(bitmap_bh); ++- ext4_error_err(sb, -err, ++- "Error %d reading block bitmap for %u", ++- err, group); ++ goto out_dbg; ++ } ++ ++ err = ext4_mb_load_buddy(sb, group, &e4b); ++ if (err) { ++- ext4_warning(sb, "Error %d loading buddy information for %u", ++- err, group); ++ put_bh(bitmap_bh); ++ goto out_dbg; ++ } ++@@ -5190,17 +5184,12 @@ repeat: ++ ++ err = ext4_mb_load_buddy_gfp(sb, group, &e4b, ++ GFP_NOFS|__GFP_NOFAIL); ++- if (err) { ++- ext4_error_err(sb, -err, "Error %d loading buddy information for %u", ++- err, group); +++ if (err) ++ return; ++- } ++ ++ bitmap_bh = ext4_read_block_bitmap(sb, group); ++ if (IS_ERR(bitmap_bh)) { ++ err = PTR_ERR(bitmap_bh); ++- ext4_error_err(sb, -err, "Error %d reading block bitmap for %u", ++- err, group); ++ ext4_mb_unload_buddy(&e4b); ++ continue; ++ } ++@@ -5495,11 +5484,8 @@ ext4_mb_discard_lg_preallocations(struct super_block *sb, ++ group = ext4_get_group_number(sb, pa->pa_pstart); ++ err = ext4_mb_load_buddy_gfp(sb, group, &e4b, ++ GFP_NOFS|__GFP_NOFAIL); ++- if (err) { ++- ext4_error_err(sb, -err, "Error %d loading buddy information for %u", ++- err, group); +++ if (err) ++ continue; ++- } ++ ext4_lock_group(sb, group); ++ list_del(&pa->pa_group_list); ++ ext4_get_group_info(sb, group)->bb_prealloc_nr--; ++@@ -5834,7 +5820,7 @@ errout: ++ * been updated or not when fail case. So can ++ * not revert pa_free back, just mark pa_error*/ ++ pa->pa_error++; ++- ext4_error(sb, +++ ext4_warning(sb, ++ "Updating bitmap error: [err %d] " ++ "[pa %p] [phy %lu] [logic %lu] " ++ "[len %u] [free %u] [error %u] " ++@@ -5845,6 +5831,7 @@ errout: ++ (unsigned)pa->pa_free, ++ (unsigned)pa->pa_error, ++ pa->pa_inode ? pa->pa_inode->i_ino : 0); +++ ext4_mark_group_bitmap_corrupted(sb, 0, 0); ++ } ++ } ++ ext4_mb_release_context(ac); ++@@ -6243,7 +6230,7 @@ do_more: ++ err = ext4_mb_load_buddy_gfp(sb, block_group, &e4b, ++ GFP_NOFS|__GFP_NOFAIL); ++ if (err) ++- goto error_return; +++ goto error_brelse; ++ ++ /* ++ * We need to make sure we don't reuse the freed block until after the ++@@ -6334,8 +6321,9 @@ do_more: ++ goto do_more; ++ } ++ error_return: ++- brelse(bitmap_bh); ++ ext4_std_error(sb, err); +++error_brelse: +++ brelse(bitmap_bh); ++ return; ++ } ++ ++@@ -6436,7 +6424,7 @@ int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, ++ ++ err = ext4_mb_load_buddy(sb, block_group, &e4b); ++ if (err) ++- goto error_return; +++ goto error_brelse; ++ ++ /* ++ * need to update group_info->bb_free and bitmap ++@@ -6475,8 +6463,9 @@ int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, ++ err = ret; ++ ++ error_return: ++- brelse(bitmap_bh); ++ ext4_std_error(sb, err); +++error_brelse: +++ brelse(bitmap_bh); ++ return err; ++ } ++ ++@@ -6594,8 +6583,6 @@ ext4_trim_all_free(struct super_block *sb, ext4_group_t group, ++ ++ ret = ext4_mb_load_buddy(sb, group, &e4b); ++ if (ret) { ++- ext4_warning(sb, "Error %d loading buddy information for %u", ++- ret, group); ++ return ret; ++ } ++ ++-- ++2.27.0 ++ +diff --git a/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series b/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series +index 621f92dfd5..e0c09a1c48 100644 +--- a/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series ++++ b/ldiskfs/kernel_patches/series/ldiskfs-5.10.0-oe2203sp1.series +@@ -14,7 +14,7 @@ rhel8/ext4-nocmtime.patch + base/ext4-htree-lock.patch + oe2203/ext4-pdirop.patch + linux-5.8/ext4-max-dir-size.patch +-linux-5.8/ext4-corrupted-inode-block-bitmaps-handling-patches.patch ++oe2203sp3/ext4-corrupted-inode-block-bitmaps-handling-patches.patch + linux-5.10/ext4-give-warning-with-dir-htree-growing.patch + ubuntu18/ext4-jcb-optimization.patch + linux-5.10/ext4-attach-jinode-in-writepages.patch +-- +2.33.0 + diff --git a/0052-LU-16541-tests-Improve-test-64f.patch b/0052-LU-16541-tests-Improve-test-64f.patch deleted file mode 100644 index be634137910bfe679b129667f6baab841ba3265c..0000000000000000000000000000000000000000 --- a/0052-LU-16541-tests-Improve-test-64f.patch +++ /dev/null @@ -1,61 +0,0 @@ -From 53e17f37ab36d4c70cff671115b6dfc5cf9b44db Mon Sep 17 00:00:00 2001 -From: Patrick Farrell -Date: Tue, 22 Aug 2023 12:32:52 -0400 -Subject: [PATCH 52/61] LU-16541 tests: Improve test 64f - -The buffered IO part of test 64f has several timing related -holes and other oddities. The use of multiop in the -background does not guarantee the RPC will not be sent, AND -the test doesn't kill it correctly. - -Clean this up and make a more reliable version of the test. -Hopefully this will resolve the failure issues, if not, a -better version of the test will allow debugging. - -Test-Parameters: trivial -Test-Parameters: testlist=sanity envdefinitions=ONLY=64f,ONLY_REPEAT=20 -Test-Parameters: clientarch=aarch64 testlist=sanity envdefinitions=ONLY=64f,ONLY_REPEAT=20 -Signed-off-by: Patrick Farrell -Change-Id: I25b825e1d9d516635ef8cbd26dd12809625c34df -Signed-off-by: Xinliang Liu ---- - lustre/tests/sanity.sh | 17 +++++++++++------ - 1 file changed, 11 insertions(+), 6 deletions(-) - -diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh -index 7b4d43ef12..c2693abd2d 100755 ---- a/lustre/tests/sanity.sh -+++ b/lustre/tests/sanity.sh -@@ -8953,18 +8953,23 @@ test_64f() { - - $LFS setstripe -c 1 -i 0 $DIR/$tfile || error "lfs setstripe failed" - -- local cmd="oO_WRONLY:w${write_bytes}_yc" -+ # Testing that buffered IO consumes grant on the client - -- $MULTIOP $DIR/$tfile $cmd & -- MULTIPID=$! -- sleep 1 -+ # Delay the RPC on the server so it's guaranteed to not complete even -+ # if the RPC is sent from the client -+ #define OBD_FAIL_PTLRPC_PAUSE_REQ 0x50a -+ $LCTL set_param fail_loc=0x50a fail_val=3 -+ dd if=/dev/zero of=$DIR/$tfile bs=$write_bytes count=1 conv=notrunc || -+ error "error writing to $DIR/$tfile with buffered IO" - - check_grants $osc_tgt $((init_grants - grants)) \ - "buffered io, not write rpc" - -- kill -USR1 $MULTIPID -- wait -+ # Clear the fail loc and do a sync on the client -+ $LCTL set_param fail_loc=0 fail_val=0 -+ sync - -+ # RPC is now known to have sent - check_grants $osc_tgt $((init_grants - grants + chunk)) \ - "buffered io, one RPC" - } --- -2.33.0 - diff --git a/0055-LU-14992-tests-add-more-mkdir_on_mdt0-calls.patch b/0055-LU-14992-tests-add-more-mkdir_on_mdt0-calls.patch deleted file mode 100644 index 8d556ba513b1bcc772cec81357a5d2269671f3c5..0000000000000000000000000000000000000000 --- a/0055-LU-14992-tests-add-more-mkdir_on_mdt0-calls.patch +++ /dev/null @@ -1,47 +0,0 @@ -From e1528526b18088a6c03af79b240e9b41e7cabfea Mon Sep 17 00:00:00 2001 -From: Mr NeilBrown -Date: Mon, 28 Nov 2022 07:49:50 +1100 -Subject: [PATCH 55/61] LU-14992 tests: add more mkdir_on_mdt0 calls - -A previous patch changed some mkdir calls in test_133a to -mkdir_on_mdt0. This allows stats collected from mdt0 to -reflect the mkdir. - -However two mkdir calls were missed, so "crossdir_rename" stats can be -wrong. - -Test-Parameters: trivial mdscount=2 mdtcount=4 testlist=sanity env=ONLY=133a - -Fixes: f0324c5c2f ("LU-14992 tests: sanity/replay-vbr mkdir on MDT0") -Signed-off-by: Mr NeilBrown -Change-Id: I4e5c2e5504307462bff4012a13ef9deb24f8da8c -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49252 -Reviewed-by: Andreas Dilger -Reviewed-by: Oleg Drokin -Reviewed-by: Jian Yu -Tested-by: jenkins -Tested-by: Maloo -Signed-off-by: Xinliang Liu -(cherry picked from commit d56ea0c80a959ebd9b393f2da048cc179cb16127) ---- - lustre/tests/sanity.sh | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh -index 3ebb944ab2..5e52f9c868 100755 ---- a/lustre/tests/sanity.sh -+++ b/lustre/tests/sanity.sh -@@ -14158,8 +14158,8 @@ test_133a() { - check_stats $SINGLEMDS "rmdir" 1 - - local testdir1=$DIR/${tdir}/stats_testdir1 -- mkdir -p ${testdir} -- mkdir -p ${testdir1} -+ mkdir_on_mdt0 -p ${testdir} -+ mkdir_on_mdt0 -p ${testdir1} - touch ${testdir1}/test1 - mv ${testdir1}/test1 ${testdir} || error "file crossdir rename" - check_stats $SINGLEMDS "crossdir_rename" 1 --- -2.33.0 - diff --git a/0056-LU-15816-tests-use-correct-ost-host-to-manage-failur.patch b/0056-LU-15816-tests-use-correct-ost-host-to-manage-failur.patch deleted file mode 100644 index afeb9a9ed0cab9f6403fe9abae1d4198cb0b352d..0000000000000000000000000000000000000000 --- a/0056-LU-15816-tests-use-correct-ost-host-to-manage-failur.patch +++ /dev/null @@ -1,72 +0,0 @@ -From d64f9f6849d46b98fc1a1fbe52484cf805baf6a4 Mon Sep 17 00:00:00 2001 -From: Mr NeilBrown -Date: Fri, 25 Nov 2022 16:13:20 +1100 -Subject: [PATCH 56/61] LU-15816 tests: use correct ost host to manage failure - -sanity test_398m sets up striping across 2 OSTs. It ensures that -failing IO to either OST individually will fail the total IO. - -However it sends the command to fail IO for the second OST (OST1) to -the host managing the first OST (ost1). If the first 2 OSTs are on -the same host, this works. If not, it fails. - -Also there error messages when testing the second stripe say "first -stripe". - -Test-Parameters: trivial env=ONLY=398m -Signed-off-by: Mr NeilBrown -Change-Id: Ic7085dab2610fa2c044a966fd8de40def0438ca4 -Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49248 -Reviewed-by: Andreas Dilger -Reviewed-by: James Simmons -Reviewed-by: Oleg Drokin -Tested-by: jenkins -Tested-by: Maloo -Signed-off-by: Xinliang Liu -(cherry picked from commit 6e66cbdb5c8c08193c36262649667747127b6d90) ---- - lustre/tests/sanity.sh | 11 ++++++----- - 1 file changed, 6 insertions(+), 5 deletions(-) - -diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh -index 5e52f9c868..76f261b2d1 100755 ---- a/lustre/tests/sanity.sh -+++ b/lustre/tests/sanity.sh -@@ -24839,6 +24839,7 @@ test_398m() { # LU-13798 - # Set up failure on OST0, the first stripe: - #define OBD_FAIL_OST_BRW_WRITE_BULK 0x20e - #NB: Fail val is ost # + 1, because we cannot use cfs_fail_val = 0 -+ # OST0 is on ost1, OST1 is on ost2. - # So this fail_val specifies OST0 - do_facet ost1 $LCTL set_param fail_loc=0x20e fail_val=1 - stack_trap "do_facet ost1 $LCTL set_param fail_loc=0" -@@ -24864,13 +24865,13 @@ test_398m() { # LU-13798 - # Clear file contents, maintain striping - echo > $DIR/$tfile - # Set up failure on OST1, second stripe: -- do_facet ost1 $LCTL set_param fail_loc=0x20e fail_val=2 -- stack_trap "do_facet ost1 $LCTL set_param fail_loc=0" -+ do_facet ost2 $LCTL set_param fail_loc=0x20e fail_val=2 -+ stack_trap "do_facet ost2 $LCTL set_param fail_loc=0" - - dd if=/dev/urandom of=$DIR/$tfile bs=8M count=8 oflag=direct && -- error "parallel dio write with failure on first stripe succeeded" -+ error "parallel dio write with failure on second stripe succeeded" - stack_trap "rm -f $DIR/$tfile" -- do_facet ost1 $LCTL set_param fail_loc=0 fail_val=0 -+ do_facet ost2 $LCTL set_param fail_loc=0 fail_val=0 - - # Place data in file for read - dd if=/dev/urandom of=$DIR/$tfile bs=8M count=8 oflag=direct || -@@ -24880,7 +24881,7 @@ test_398m() { # LU-13798 - #define OBD_FAIL_OST_BRW_READ_BULK 0x20f - do_facet ost2 $LCTL set_param fail_loc=0x20f fail_val=2 - dd if=$DIR/$tfile of=$DIR/$tfile.2 bs=8M count=8 iflag=direct && -- error "parallel dio read with error on first stripe succeeded" -+ error "parallel dio read with error on second stripe succeeded" - rm -f $DIR/$tfile.2 - do_facet ost2 $LCTL set_param fail_loc=0 fail_val=0 - } --- -2.33.0 - diff --git a/2.15.3.tar.gz b/2.15.4.tar.gz similarity index 77% rename from 2.15.3.tar.gz rename to 2.15.4.tar.gz index 68890f571280382a1d7c33d6fcff991bce89cb7a..7a8160f92141a397d3e4ad2766e86f7613689e42 100644 Binary files a/2.15.3.tar.gz and b/2.15.4.tar.gz differ diff --git a/lustre.spec b/lustre.spec index 3757c249a48d37f1fb725370f4e1bb8c3f071aef..70988bebe784426adca8d1cb1b88d95641bf7cc3 100644 --- a/lustre.spec +++ b/lustre.spec @@ -18,6 +18,7 @@ %bcond_with mofed %bcond_with mpi %bcond_with kabi +%bcond_without l_getsepol # LUTF Turn off brp-python-precompile script as we don't want the python files # to be compiled on installation @@ -49,7 +50,7 @@ %undefine with_lustre_tests %endif -%{!?version: %global version 2.15.3} +%{!?version: %global version 2.15.4} # if you want a custom kernel version set it variable with $ver.$arch %{!?kver: %global kver %(rpm -q --qf '%%{VERSION}-%%{RELEASE}.%%{ARCH}' `rpm -q kernel-devel | sort -rV|head -n 1`)} # cut epoch for kmodtool @@ -184,7 +185,7 @@ Summary: Lustre File System Name: %{lustre_name} -Version: 2.15.3 +Version: 2.15.4 Release: 1 License: GPL-2.0-only AND LGPL-2.1-or-later Group: System Environment/Kernel @@ -203,67 +204,55 @@ URL: https://wiki.whamcloud.com/ BuildRoot: %{_tmppath}/lustre-%{version}-root # patches -Patch01: 0001-Prepare-for-next-pointrelease.patch -Patch02: 0002-LU-15821-ldlm-Prioritize-blocking-callbacks.patch -Patch03: 0003-LU-14377-tests-make-parallel-scale-rr_alloc-less-str.patch -Patch04: 0004-LU-15123-tests-check-quota-reintegration-after-recov.patch -Patch05: 0005-LU-13081-tests-skip-sanity-test_151-test_156.patch -Patch06: 0006-LU-11785-tests-fix-conf-sanity-98-mount-check-on-64K.patch -Patch07: 0007-LU-11388-tests-replay-single-131b-to-refresh-grants.patch -Patch08: 0008-LU-16163-tests-skip-racer_on_nfs-for-NFSv3.patch -Patch09: 0009-LU-14294-tests-fixed-NFS-configuration-issue.patch -Patch10: 0010-LU-16717-mdt-treat-unknown-hash-type-as-sane-type.patch -Patch11: 0011-LU-15481-llog-Add-LLOG_SKIP_PLAIN-to-skip-llog-plain.patch -Patch12: 0012-LU-6612-utils-strengthen-llog_reader-vs-wrong-format.patch -Patch13: 0013-LU-16052-llog-handle-EBADR-for-catalog-processing.patch -Patch14: 0014-LU-16717-mdt-resume-dir-migration-with-bad_type.patch -Patch15: 0015-LU-14668-lnet-Lock-primary-NID-logic.patch -Patch16: 0016-LU-14668-lnet-Peers-added-via-kernel-API-should-be-p.patch -Patch17: 0017-LU-14668-lnet-don-t-delete-peer-created-by-Lustre.patch -Patch18: 0018-LU-14668-lnet-add-force-option-to-lnetctl-peer-del.patch -Patch19: 0019-LU-14668-lnet-add-lock_prim_nid-lnet-module-paramete.patch -Patch20: 0020-LU-14668-tests-verify-state-of-peer-added-with-lock_.patch -Patch21: 0021-LU-11787-test-Fix-checkfilemap-tests-for-64K-page.patch -Patch22: 0022-LU-15800-ofd-take-a-read-lock-for-fallocate.patch -Patch23: 0023-LU-16873-osd-update-OI_Scrub-file-with-new-magic.patch -Patch24: 0024-LU-15519-quota-fallocate-does-not-increase-projectid.patch -Patch25: 0025-LU-16060-osd-ldiskfs-copy-nul-byte-terminator-in-wri.patch -Patch26: 0026-LU-16934-kernel-update-RHEL-8.8-4.18.0-477.15.1.el8_.patch -Patch27: 0027-LU-15740-tests-scale-fs_log_size-by-OSTCOUNT.patch -Patch28: 0028-LU-16943-tests-fix-replay-single-135-under-hard-fail.patch -Patch29: 0029-LU-16517-build-pass-extra-configure-options-to-make-.patch -Patch30: 0030-LU-15193-quota-expand-QUOTA_MAX_TRANSIDS-to-12.patch -Patch31: 0031-LU-16916-tests-fix-client_evicted-not-to-ignore-EOPN.patch -Patch32: 0032-LU-16626-build-remove-python2-dependencies.patch -Patch33: 0033-LU-16943-tests-use-primary-ost1-server-in-replay-sin.patch -Patch34: 0034-LU-16585-build-remove-python2-dependencies.patch -Patch35: 0035-LU-15660-statahead-statahead-thread-doesn-t-stop.patch -Patch36: 0036-LU-16042-tests-can-not-get-cache-size-on-Arm64.patch -Patch37: 0037-LU-16662-autoconf-fix-configure-test-compile-for-CON.patch -Patch38: 0038-LU-16322-build-Add-client-build-support-for-openEule.patch -Patch39: 0039-LU-16481-build-add-server-support-for-openEuler.patch -Patch40: 0040-LU-16824-ldiskfs-add-support-for-openEuler-22.03-LTS.patch -Patch41: 0041-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP2.patch -Patch42: 0042-lustre.spec.in-match-rpm-macro-openEuler-for-openEul.patch -Patch43: 0043-LU-15722-osd-ldiskfs-fix-IO-write-gets-stuck-for-64K.patch -Patch44: 0044-LU-15722-osd-ldiskfs-fix-write-stuck-for-64K-PAGE_SI.patch -Patch45: 0045-LU-15978-osp-fix-striped-directory-deletion-fails-fo.patch -Patch46: 0046-ldiskfs-add-support-for-oe2003.patch -Patch47: 0047-lustre.spec.in-Add-gcc-option-Wno-stringop-overflow.patch -Patch48: 0048-LU-16321-osd-Allow-fiemap-on-kernel-buffers.patch -Patch49: 0049-LU-13135-quota-improve-checks-in-OSDs-to-ignore-quot.patch -Patch50: 0050-LU-16893-libcfs-Remove-force_sig-usage-from-lfsck.patch -Patch51: 0051-LU-16534-build-Prefer-timer_delete-_sync.patch -Patch52: 0052-LU-16541-tests-Improve-test-64f.patch -Patch53: 0053-LU-16788-tests-sanity-should-remove-temp-files.patch -Patch54: 0054-LU-14992-tests-sanity-replay-vbr-mkdir-on-MDT0.patch -Patch55: 0055-LU-14992-tests-add-more-mkdir_on_mdt0-calls.patch -Patch56: 0056-LU-15816-tests-use-correct-ost-host-to-manage-failur.patch -Patch57: 0057-LU-16571-utils-fix-parallel-lfs-migrate-b-on-hard-li.patch -Patch58: 0058-LU-14073-ldiskfs-don-t-test-LDISKFS_IOC_FSSETXATTR.patch -Patch59: 0059-LU-16019-llite-fully-disable-readahead-in-kernel-I-O.patch -Patch60: 0060-Update-openEuler-22.03-kernels.patch -Patch61: 0061-Update-kernel-for-openEuler-20.03-LTS.patch +Patch01: 0001-LU-15658-lod-ost-list-and-pool-name-conflict.patch +Patch02: 0002-LU-16894-lod-fix-stripe_count-limit-in-lod_qos_set_p.patch +Patch03: 0003-LU-15595-tests-Add-various-router-tests.patch +Patch04: 0004-LU-14073-ldiskfs-don-t-test-LDISKFS_IOC_FSSETXATTR.patch +Patch05: 0005-LU-14824-test-sanity-413a-b-unlink-timeout-v2.patch +Patch06: 0006-LU-15207-libcfs-reset-hs_rehash_bits.patch +Patch07: 0007-LU-16076-utils-enhance-lfs-check-command.patch +Patch08: 0008-LU-16057-obdclass-set-OBD_MD_FLGROUP-for-ladvise-RPC.patch +Patch09: 0009-LU-16772-quota-protect-lqe_glbl_data-in-qmt_site_rec.patch +Patch10: 0010-LU-13135-quota-improve-checks-in-OSDs-to-ignore-quot.patch +Patch11: 0011-LU-16019-llite-fully-disable-readahead-in-kernel-I-O.patch +Patch12: 0012-LU-15655-contrib-update-branch_comm-to-python3.patch +Patch13: 0013-LU-16101-tests-skip-sanity-27J-for-more-kernels.patch +Patch14: 0014-LU-17175-tests-fix-sanity-krb5-test_5-in-interop-mod.patch +Patch15: 0015-LU-14992-tests-sanity-replay-vbr-mkdir-on-MDT0.patch +Patch16: 0016-LU-17226-build-create-config-option-for-l_getsepol.patch +Patch17: 0017-LU-17664-lnet-LNetPrimaryNID-returns-wrong-NID.patch +Patch18: 0018-LU-16351-llite-Linux-6.1-prandom-folios_contig-vma_i.patch +Patch19: 0019-LU-10680-tests-fix-interop-for-sanity-test_160h.patch +Patch20: 0020-LU-17409-scripts-correct-ldev-MGS-handling.patch +Patch21: 0021-LU-15947-obdclass-improve-precision-of-wakeups-for-m.patch +Patch22: 0022-LU-17510-obdclass-fix-wake-up-when-queuing-close-req.patch +Patch23: 0023-LU-15851-lnet-Adjust-niov-checks-for-large-MD.patch +Patch24: 0024-LU-17010-lfsck-don-t-dump-stack-repeatedly.patch +Patch25: 0025-LU-16502-lutf-fix-bugs-in-bash-scripts.patch +Patch26: 0026-LU-16382-config-ensure-lutf.sh-is-included-in-dist.patch +Patch27: 0027-LU-15515-contrib-epython-scripts-to-python3-conversi.patch +Patch28: 0028-LU-16322-build-Add-client-build-support-for-openEule.patch +Patch29: 0029-LU-16481-build-add-server-support-for-openEuler.patch +Patch30: 0030-LU-16824-ldiskfs-add-support-for-openEuler-22.03-LTS.patch +Patch31: 0031-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP2.patch +Patch32: 0032-lustre.spec.in-match-rpm-macro-openEuler-for-openEul.patch +Patch33: 0033-LU-15722-osd-ldiskfs-fix-IO-write-gets-stuck-for-64K.patch +Patch34: 0034-LU-15722-osd-ldiskfs-fix-write-stuck-for-64K-PAGE_SI.patch +Patch35: 0035-LU-15978-osp-fix-striped-directory-deletion-fails-fo.patch +Patch36: 0036-ldiskfs-add-support-for-oe2003.patch +Patch37: 0037-lustre.spec.in-Add-gcc-option-Wno-stringop-overflow.patch +Patch38: 0038-LU-16321-osd-Allow-fiemap-on-kernel-buffers.patch +Patch39: 0039-LU-16893-libcfs-Remove-force_sig-usage-from-lfsck.patch +Patch40: 0040-LU-16534-build-Prefer-timer_delete-_sync.patch +Patch41: 0041-LU-16788-tests-sanity-should-remove-temp-files.patch +Patch42: 0042-LU-16571-utils-fix-parallel-lfs-migrate-b-on-hard-li.patch +Patch43: 0043-Update-openEuler-22.03-kernels.patch +Patch44: 0044-Update-kernel-for-openEuler-20.03-LTS.patch +Patch45: 0045-ldiskfs-fix-build-for-oe2203sp3.patch +Patch46: 0046-Fix-sanity-running-stuck.patch +Patch47: 0047-LU-16976-ldiskfs-add-support-for-openEuler-22.03-SP3.patch +Patch48: 0048-ldiskfs-fix-build-for-kernel-5.10.0-186.0.0.99.oe220.patch +Patch49: 0049-Fix-build-for-5.10.0-197.0.0.110.oe2203sp3.patch %if %{with lustre_modules} Requires: %{requires_kmod_name} = %{requires_kmod_version} @@ -300,6 +289,9 @@ Provides: lustre-client = %{version}-%{release} #suse don't support selinux BuildRequires: libselinux-devel %endif +%if %{with l_getsepol} +BuildRequires: pkgconfig(libselinux) openssl-devel +%endif %if %{with lustre_modules} %if %{with mofed} BuildRequires: mlnx-ofa_kernel-devel @@ -615,6 +607,7 @@ fi %{!?with_systemd:--with-systemdsystemunitdir=no} \ %{?with_systemd:--with-systemdsystemunitdir=%{_unitdir}} \ %{?with_mofed:--with-o2ib=$o2ib_path} \ + %{!?with_l_getsepol:--disable-l_getsepol} \ --with-linux=%{kdir} \ --with-linux-obj=%{kobjdir} \ --with-kmp-moddir=%{kmoddir}/%{name} @@ -937,6 +930,10 @@ rm -rf $RPM_BUILD_ROOT rm -rf %{_tmppath}/kmp %changelog +* Tue May 21 2024 Xinliang Liu - 2.15.4-1 +- Fix ldiskfs build. +- Update to 2.15.4. + * Fri Nov 10 2023 Xinliang Liu - 2.15.3-1 - Update to 2.15.3 with openEuler bugfixes patches.