lkml.org 
[lkml]   [2019]   [Dec]   [30]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v8 4/5] locking/qspinlock: Introduce starvation avoidance into CNA
    Date
    Keep track of the number of intra-node lock handoffs, and force
    inter-node handoff once this number reaches a preset threshold.
    The default value for the threshold can be overridden with
    the new kernel boot command-line option "numa_spinlock_threshold".

    Signed-off-by: Alex Kogan <alex.kogan@oracle.com>
    Reviewed-by: Steve Sistare <steven.sistare@oracle.com>
    ---
    .../admin-guide/kernel-parameters.txt | 8 ++++
    kernel/locking/qspinlock.c | 3 ++
    kernel/locking/qspinlock_cna.h | 41 ++++++++++++++++++-
    3 files changed, 51 insertions(+), 1 deletion(-)

    diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
    index b68cb80e477f..30d79819a3b0 100644
    --- a/Documentation/admin-guide/kernel-parameters.txt
    +++ b/Documentation/admin-guide/kernel-parameters.txt
    @@ -3200,6 +3200,14 @@
    Not specifying this option is equivalent to
    numa_spinlock=auto.

    + numa_spinlock_threshold= [NUMA, PV_OPS]
    + Set the threshold for the number of intra-node
    + lock hand-offs before the NUMA-aware spinlock
    + is forced to be passed to a thread on another NUMA node.
    + Valid values are in the [0..31] range. Smaller values
    + result in a more fair, but less performant spinlock, and
    + vice versa. The default value is 16.
    +
    cpu0_hotplug [X86] Turn on CPU0 hotplug feature when
    CONFIG_BOOTPARAM_HOTPLUG_CPU0 is off.
    Some features depend on CPU0. Known dependencies are:
    diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
    index 609980a53841..e382d8946ccc 100644
    --- a/kernel/locking/qspinlock.c
    +++ b/kernel/locking/qspinlock.c
    @@ -597,6 +597,9 @@ EXPORT_SYMBOL(queued_spin_lock_slowpath);
    #if !defined(_GEN_CNA_LOCK_SLOWPATH) && defined(CONFIG_NUMA_AWARE_SPINLOCKS)
    #define _GEN_CNA_LOCK_SLOWPATH

    +#undef pv_init_node
    +#define pv_init_node cna_init_node
    +
    #undef pv_wait_head_or_lock
    #define pv_wait_head_or_lock cna_pre_scan

    diff --git a/kernel/locking/qspinlock_cna.h b/kernel/locking/qspinlock_cna.h
    index 3c99a4a6184b..30feff02865d 100644
    --- a/kernel/locking/qspinlock_cna.h
    +++ b/kernel/locking/qspinlock_cna.h
    @@ -51,13 +51,25 @@ struct cna_node {
    int numa_node;
    u32 encoded_tail;
    u32 pre_scan_result; /* encoded tail or enum val */
    + u32 intra_count;
    };

    enum {
    LOCAL_WAITER_FOUND = 2, /* 0 and 1 are reserved for @locked */
    + FLUSH_SECONDARY_QUEUE = 3,
    MIN_ENCODED_TAIL
    };

    +/*
    + * Controls the threshold for the number of intra-node lock hand-offs before
    + * the NUMA-aware variant of spinlock is forced to be passed to a thread on
    + * another NUMA node. By default, the chosen value provides reasonable
    + * long-term fairness without sacrificing performance compared to a lock
    + * that does not have any fairness guarantees. The default setting can
    + * be changed with the "numa_spinlock_threshold" boot option.
    + */
    +int intra_node_handoff_threshold __ro_after_init = 1 << 16;
    +
    static void __init cna_init_nodes_per_cpu(unsigned int cpu)
    {
    struct mcs_spinlock *base = per_cpu_ptr(&qnodes[0].mcs, cpu);
    @@ -97,6 +109,11 @@ static int __init cna_init_nodes(void)
    }
    early_initcall(cna_init_nodes);

    +static __always_inline void cna_init_node(struct mcs_spinlock *node)
    +{
    + ((struct cna_node *)node)->intra_count = 0;
    +}
    +
    /* this function is called only when the primary queue is empty */
    static inline bool cna_try_change_tail(struct qspinlock *lock, u32 val,
    struct mcs_spinlock *node)
    @@ -233,7 +250,9 @@ __always_inline u32 cna_pre_scan(struct qspinlock *lock,
    {
    struct cna_node *cn = (struct cna_node *)node;

    - cn->pre_scan_result = cna_scan_main_queue(node, node);
    + cn->pre_scan_result =
    + cn->intra_count == intra_node_handoff_threshold ?
    + FLUSH_SECONDARY_QUEUE : cna_scan_main_queue(node, node);

    return 0;
    }
    @@ -263,6 +282,9 @@ static inline void cna_pass_lock(struct mcs_spinlock *node,
    * if we acquired the MCS lock when its queue was empty
    */
    val = node->locked ? node->locked : 1;
    + /* inc @intra_count if the secondary queue is not empty */
    + ((struct cna_node *)next_holder)->intra_count =
    + cn->intra_count + (node->locked > 1);
    } else if (node->locked > 1) { /* if secondary queue is not empty */
    /* next holder will be the first node in the secondary queue */
    tail_2nd = decode_tail(node->locked);
    @@ -317,3 +339,20 @@ void cna_configure_spin_lock_slowpath(void)
    pr_info("Enabling CNA spinlock\n");
    }
    }
    +
    +static int __init numa_spinlock_threshold_setup(char *str)
    +{
    + int new_threshold_param;
    +
    + if (get_option(&str, &new_threshold_param)) {
    + /* valid value is between 0 and 31 */
    + if (new_threshold_param < 0 || new_threshold_param > 31)
    + return 0;
    +
    + intra_node_handoff_threshold = 1 << new_threshold_param;
    + return 1;
    + }
    +
    + return 0;
    +}
    +__setup("numa_spinlock_threshold=", numa_spinlock_threshold_setup);
    --
    2.21.0 (Apple Git-122.2)
    \
     
     \ /
      Last update: 2019-12-30 20:54    [W:2.037 / U:0.360 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site