Series comparison

-[PATCH mptcp-next v1 0/6] BPF path manager, part 4
+[PATCH mptcp-next v3 0/3] BPF path manager, part 4
 From: Geliang Tang <tanggeliang@kylinos.cn>
-Some path manager related refactoring and cleanups.
+v3:
  - rename the 2nd parameter of get_local_id() from 'local' to 'skc'.
  - keep the 'msk_sport' check in mptcp_userspace_pm_get_local_id().
  - return 'err' instead of '0' in userspace_pm_subflow_create().
  - drop 'ret' variable inmptcp_pm_data_reset().
  - fix typos in commit log.
-Geliang Tang (6):
+Depends on: "BPF path manager, part 3" v4
-  mptcp: pm: in-kernel: avoid access entry without lock
+Based-on: <cover.1737012165.git.tanggeliang@kylinos.cn>
   mptcp: pm: in-kernel: reduce parameters of set_flags
   mptcp: pm: use addr entry for get_local_id
   mptcp: pm: in-kernel: use kmemdup helper
   sock: add sock_kmemdup helper
   mptcp: pm: userspace: use sock_kmemdup helper
- include/net/sock.h       |  1 +
+v2:
- net/core/sock.c          | 23 +++++++++++++++++++++++
+ - update get_local_id interface in patch 2.
- net/mptcp/pm.c           |  9 ++++++---
- net/mptcp/pm_netlink.c   | 30 +++++++++++++-----------------
+get_addr() and dump_addr() interfaces of BPF userspace pm are dropped
- net/mptcp/pm_userspace.c | 20 +++++++-------------
+as Matt suggested.
- net/mptcp/protocol.h     |  6 ++++--
-files changed, 54 insertions(+), 35 deletions(-)
+In order to implement BPF userspace path manager, it is necessary to
 unify the interfaces of the path manager. This set contains some
 cleanups and refactoring to unify the interfaces in kernel space.
 Finally, define a struct mptcp_pm_ops for a userspace path manager
 like this:
 struct mptcp_pm_ops {
     int (*address_announce)(struct mptcp_sock *msk,
                 struct mptcp_pm_addr_entry *local);
     int (*address_remove)(struct mptcp_sock *msk, u8 id);
     int (*subflow_create)(struct mptcp_sock *msk,
                   struct mptcp_pm_addr_entry *local,
                   struct mptcp_addr_info *remote);
     int (*subflow_destroy)(struct mptcp_sock *msk,
                    struct mptcp_pm_addr_entry *local,
                    struct mptcp_addr_info *remote);
     int (*get_local_id)(struct mptcp_sock *msk,
                 struct mptcp_pm_addr_entry *skc);
     u8 (*get_flags)(struct mptcp_sock *msk,
             struct mptcp_addr_info *skc);
     int (*set_flags)(struct mptcp_sock *msk,
              struct mptcp_pm_addr_entry *local,
              struct mptcp_addr_info *remote);
     u8            type;
     struct module        *owner;
     struct list_head    list;
     void (*init)(struct mptcp_sock *msk);
     void (*release)(struct mptcp_sock *msk);
 } ____cacheline_aligned_in_smp;
 Closes: https://github.com/multipath-tcp/mptcp_net-next/issues/74
 Geliang Tang (3):
   mptcp: define struct mptcp_pm_ops
   mptcp: register default userspace pm
   mptcp: init and release mptcp_pm_ops
  include/net/mptcp.h      |  27 +++
  net/mptcp/pm.c           |   5 +
  net/mptcp/pm_userspace.c | 374 ++++++++++++++++++++++++++++-----------
  net/mptcp/protocol.c     |   1 +
  net/mptcp/protocol.h     |   9 +
 files changed, 313 insertions(+), 103 deletions(-)
 --
 .43.0

-[PATCH mptcp-next v1 1/6] mptcp: pm: in-kernel: avoid access entry without lock
+Deleted patch
-From: Geliang Tang <tanggeliang@kylinos.cn>
-In mptcp_pm_nl_set_flags(), "entry" is copied to "local" when pernet->lock
-is held to avoid direct access to entry without pernet->lock.
-Therefore, "local->flags" should be passed to mptcp_nl_set_flags instead
-of "entry->flags" when pernet->lock is not held, so as to avoid access to
-entry.
-Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
----
- net/mptcp/pm_netlink.c | 2 +-
-file changed, 1 insertion(+), 1 deletion(-)
-diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/mptcp/pm_netlink.c
-+++ b/net/mptcp/pm_netlink.c
-@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_set_flags(struct mptcp_pm_addr_entry *local,
-     *local = *entry;
-     spin_unlock_bh(&pernet->lock);
--    mptcp_nl_set_flags(net, &local->addr, entry->flags, changed);
-+    mptcp_nl_set_flags(net, &local->addr, local->flags, changed);
-     return 0;
- }
---
-.43.0

-[PATCH mptcp-next v1 2/6] mptcp: pm: in-kernel: reduce parameters of set_flags
+Deleted patch
-From: Geliang Tang <tanggeliang@kylinos.cn>
-The number of parameters in mptcp_nl_set_flags() can be reduced.
-Only need to pass a "local" parameter to it instead of "local->addr"
-and "local->flags".
-Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
----
- net/mptcp/pm_netlink.c | 15 ++++++++-------
-file changed, 8 insertions(+), 7 deletions(-)
-diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/mptcp/pm_netlink.c
-+++ b/net/mptcp/pm_netlink.c
-@@ -XXX,XX +XXX,XX @@ static void mptcp_pm_nl_fullmesh(struct mptcp_sock *msk,
-     spin_unlock_bh(&msk->pm.lock);
- }
--static void mptcp_nl_set_flags(struct net *net, struct mptcp_addr_info *addr,
--                   u8 flags, u8 changed)
-+static void mptcp_nl_set_flags(struct net *net,
-+                   struct mptcp_pm_addr_entry *local,
-+                   u8 changed)
- {
--    u8 is_subflow = !!(flags & MPTCP_PM_ADDR_FLAG_SUBFLOW);
--    u8 bkup = !!(flags & MPTCP_PM_ADDR_FLAG_BACKUP);
-+    u8 is_subflow = !!(local->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW);
-+    u8 bkup = !!(local->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
-     long s_slot = 0, s_num = 0;
-     struct mptcp_sock *msk;
-@@ -XXX,XX +XXX,XX @@ static void mptcp_nl_set_flags(struct net *net, struct mptcp_addr_info *addr,
-         lock_sock(sk);
-         if (changed & MPTCP_PM_ADDR_FLAG_BACKUP)
--            mptcp_pm_nl_mp_prio_send_ack(msk, addr, NULL, bkup);
-+            mptcp_pm_nl_mp_prio_send_ack(msk, &local->addr, NULL, bkup);
-         /* Subflows will only be recreated if the SUBFLOW flag is set */
-         if (is_subflow && (changed & MPTCP_PM_ADDR_FLAG_FULLMESH))
--            mptcp_pm_nl_fullmesh(msk, addr);
-+            mptcp_pm_nl_fullmesh(msk, &local->addr);
-         release_sock(sk);
- next:
-@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_set_flags(struct mptcp_pm_addr_entry *local,
-     *local = *entry;
-     spin_unlock_bh(&pernet->lock);
--    mptcp_nl_set_flags(net, &local->addr, local->flags, changed);
-+    mptcp_nl_set_flags(net, local, changed);
-     return 0;
- }
---
-.43.0

-[PATCH mptcp-next v1 6/6] mptcp: pm: userspace: use sock_kmemdup helper
+[PATCH mptcp-next v3 1/3] mptcp: define struct mptcp_pm_ops
 From: Geliang Tang <tanggeliang@kylinos.cn>
-Instead of using sock_kmalloc() to allocate an entry and then
+In order to allow users to develop their own BPF-based path manager,
-immediately duplicate another entry to the newly allocated one,
+this patch defines a struct ops "mptcp_pm_ops" for a userspace path
-sock_kmemdup() helper can be used to simplify the code.
+manager, which contains a set of interfaces.
-More importantly, the code "*e = *entry;" that assigns "entry"
+Add a set of functions to register, unregister, find and validate a
-to "e" is not easy to implemented in BPF if we use the same code
+given struct ops.
 to implement an append_new_local_addr() helper of a BFP path
 manager. This patch avoids this type of memory assignment
 operation.
 Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
 ---
- net/mptcp/pm_userspace.c | 3 +--
+ include/net/mptcp.h      | 27 +++++++++++++++++++
-file changed, 1 insertion(+), 2 deletions(-)
+ net/mptcp/pm_userspace.c | 57 ++++++++++++++++++++++++++++++++++++++++
  net/mptcp/protocol.h     |  5 ++++
 files changed, 89 insertions(+)
+diff --git a/include/net/mptcp.h b/include/net/mptcp.h
+index XXXXXXX..XXXXXXX 100644
+--- a/include/net/mptcp.h
++++ b/include/net/mptcp.h
+@@ -XXX,XX +XXX,XX @@
+ struct mptcp_info;
+ struct mptcp_sock;
++struct mptcp_pm_addr_entry;
+ struct seq_file;
+ /* MPTCP sk_buff extension data */
+@@ -XXX,XX +XXX,XX @@ struct mptcp_sched_ops {
+     void (*release)(struct mptcp_sock *msk);
+ } ____cacheline_aligned_in_smp;
++struct mptcp_pm_ops {
++    int (*address_announce)(struct mptcp_sock *msk,
++                struct mptcp_pm_addr_entry *local);
++    int (*address_remove)(struct mptcp_sock *msk, u8 id);
++    int (*subflow_create)(struct mptcp_sock *msk,
++                  struct mptcp_pm_addr_entry *local,
++                  struct mptcp_addr_info *remote);
++    int (*subflow_destroy)(struct mptcp_sock *msk,
++                   struct mptcp_pm_addr_entry *local,
++                   struct mptcp_addr_info *remote);
++    int (*get_local_id)(struct mptcp_sock *msk,
++                struct mptcp_pm_addr_entry *skc);
++    u8 (*get_flags)(struct mptcp_sock *msk,
++            struct mptcp_addr_info *skc);
++    int (*set_flags)(struct mptcp_sock *msk,
++             struct mptcp_pm_addr_entry *local,
++             struct mptcp_addr_info *remote);
++
++    u8            type;
++    struct module        *owner;
++    struct list_head    list;
++
++    void (*init)(struct mptcp_sock *msk);
++    void (*release)(struct mptcp_sock *msk);
++} ____cacheline_aligned_in_smp;
++
+ #ifdef CONFIG_MPTCP
+ void mptcp_init(void);
 diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/mptcp/pm_userspace.c
 +++ b/net/mptcp/pm_userspace.c
-@@ -XXX,XX +XXX,XX @@ static int mptcp_userspace_pm_append_new_local_addr(struct mptcp_sock *msk,
+@@ -XXX,XX +XXX,XX @@
-         /* Memory for the entry is allocated from the
+  * Copyright (c) 2022, Intel Corporation.
-          * sock option buffer.
+  */
-          */
--        e = sock_kmalloc(sk, sizeof(*e), GFP_ATOMIC);
++#include <linux/rculist.h>
-+        e = sock_kmemdup(sk, entry, sizeof(*entry), GFP_ATOMIC);
++#include <linux/spinlock.h>
-         if (!e) {
+ #include "protocol.h"
-             ret = -ENOMEM;
+ #include "mib.h"
-             goto append_err;
+ #include "mptcp_pm_gen.h"
-         }
+@@ -XXX,XX +XXX,XX @@
+     list_for_each_entry(__entry,                        \
--        *e = *entry;
+                 &((__msk)->pm.userspace_pm_local_addr_list), list)
-         if (!e->addr.id && needs_id)
-             e->addr.id = find_next_zero_bit(id_bitmap,
++static DEFINE_SPINLOCK(mptcp_pm_list_lock);
-                             MPTCP_PM_MAX_ADDR_ID + 1,
++static LIST_HEAD(mptcp_pm_list);
 +
  void mptcp_free_local_addr_list(struct mptcp_sock *msk)
  {
      struct mptcp_pm_addr_entry *entry, *tmp;
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_get_addr(u8 id, struct mptcp_pm_addr_entry *addr,
      sock_put(sk);
      return ret;
  }
 +
 +/* Must be called with rcu read lock held */
 +struct mptcp_pm_ops *mptcp_pm_find(enum mptcp_pm_type type)
 +{
 +    struct mptcp_pm_ops *pm;
 +
 +    list_for_each_entry_rcu(pm, &mptcp_pm_list, list) {
 +        if (pm->type == type)
 +            return pm;
 +    }
 +
 +    return NULL;
 +}
 +
 +int mptcp_validate_path_manager(struct mptcp_pm_ops *pm)
 +{
 +    if (!pm->address_announce && !pm->address_remove &&
 +        !pm->subflow_create && !pm->subflow_destroy &&
 +        !pm->get_local_id && !pm->get_flags && !pm->set_flags) {
 +        pr_err("%u does not implement required ops\n", pm->type);
 +        return -EINVAL;
 +    }
 +
 +    return 0;
 +}
 +
 +int mptcp_register_path_manager(struct mptcp_pm_ops *pm)
 +{
 +    int ret;
 +
 +    ret = mptcp_validate_path_manager(pm);
 +    if (ret)
 +        return ret;
 +
 +    spin_lock(&mptcp_pm_list_lock);
 +    if (mptcp_pm_find(pm->type)) {
 +        spin_unlock(&mptcp_pm_list_lock);
 +        return -EEXIST;
 +    }
 +    list_add_tail_rcu(&pm->list, &mptcp_pm_list);
 +    spin_unlock(&mptcp_pm_list_lock);
 +
 +    pr_debug("userspace_pm type %u registered\n", pm->type);
 +    return 0;
 +}
 +
 +void mptcp_unregister_path_manager(struct mptcp_pm_ops *pm)
 +{
 +    spin_lock(&mptcp_pm_list_lock);
 +    list_del_rcu(&pm->list);
 +    spin_unlock(&mptcp_pm_list_lock);
 +}
 diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
 index XXXXXXX..XXXXXXX 100644
 --- a/net/mptcp/protocol.h
 +++ b/net/mptcp/protocol.h
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_
  void mptcp_pm_remove_addr_entry(struct mptcp_sock *msk,
                  struct mptcp_pm_addr_entry *entry);
 +struct mptcp_pm_ops *mptcp_pm_find(enum mptcp_pm_type type);
 +int mptcp_validate_path_manager(struct mptcp_pm_ops *pm);
 +int mptcp_register_path_manager(struct mptcp_pm_ops *pm);
 +void mptcp_unregister_path_manager(struct mptcp_pm_ops *pm);
 +
  void mptcp_free_local_addr_list(struct mptcp_sock *msk);
  void mptcp_event(enum mptcp_event_type type, const struct mptcp_sock *msk,
 --
 .43.0

-[PATCH mptcp-next v1 3/6] mptcp: pm: use addr entry for get_local_id
+[PATCH mptcp-next v3 2/3] mptcp: register default userspace pm
 From: Geliang Tang <tanggeliang@kylinos.cn>
-The following code in mptcp_userspace_pm_get_local_id() that assigns "skc"
+This patch defines the original userspace pm as the default path manager,
-to "new_entry" is not allowed in BPF if we use the same code to implement
+named mptcp_userspace_pm, and register it in mptcp_pm_data_init().
-the get_local_id() interface of a BFP path manager:
+Extract address_announce() interface of the userspace PM from the handler
-    memset(&new_entry, 0, sizeof(struct mptcp_pm_addr_entry));
+of netlink command MPTCP_PM_CMD_ANNOUNCE mptcp_pm_nl_announce_doit(), only
-    new_entry.addr = *skc;
+leave the code for obtaining msk through "info" and parsing address entry
-    new_entry.addr.id = 0;
+in the handler.
-    new_entry.flags = MPTCP_PM_ADDR_FLAG_IMPLICIT;
+Extract address_remove() interface of the userspace PM from the handler
-To solve the issue, this patch moves this assignment to "new_entry" forward
+of netlink command MPTCP_PM_CMD_REMOVE mptcp_pm_nl_remove_doit(), only
-to mptcp_pm_get_local_id(), and then passing "new_entry" as a parameter to
+leave the code for parsing address id and obtaining msk through "info"
-both mptcp_pm_nl_get_local_id() and mptcp_userspace_pm_get_local_id().
+in the handler.
 Extract subflow_create() interface of the userspace PM from the handler of
 netlink command MPTCP_PM_CMD_SUBFLOW_CREATE
     mptcp_pm_nl_subflow_create_doit(),
 only leave the code for obtaining msk through "info", parsing local address
 entry and parsing remote address info in the handler.
 Extract subflow_destroy() interface of the userspace PM from the handler of
 netlink command MPTCP_PM_CMD_SUBFLOW_DESTROY
     mptcp_pm_nl_subflow_destroy_doit(),
 only leave the code for obtaining msk through "info", parsing local address
 entry and parsing remote address info in the handler.
 Extract set_flags() interface of the userspace PM from function
 mptcp_userspace_pm_set_flags(), only leave the code for obtaining
 msk through "info" in this function.
 All fives interfaces are invoked under holding the msk socket lock.
 Extract get_local_id() interface of the userspace PM from function
 mptcp_userspace_pm_get_local_id(), only leave the code for setting
 new address entry in this function.
 get_flags() interface of the userspace PM is defined as the same as
 the function mptcp_userspace_pm_get_flags(), then this function
 becomes a wrapper.
 Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
 ---
- net/mptcp/pm.c           |  9 ++++++---
+ net/mptcp/pm.c           |   1 +
- net/mptcp/pm_netlink.c   | 11 ++++-------
+ net/mptcp/pm_userspace.c | 273 ++++++++++++++++++++++++---------------
- net/mptcp/pm_userspace.c | 17 ++++++-----------
+ net/mptcp/protocol.h     |   1 +
- net/mptcp/protocol.h     |  6 ++++--
+files changed, 172 insertions(+), 103 deletions(-)
 files changed, 20 insertions(+), 23 deletions(-)
 diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/mptcp/pm.c
 +++ b/net/mptcp/pm.c
-@@ -XXX,XX +XXX,XX @@ bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
+@@ -XXX,XX +XXX,XX @@ void mptcp_pm_data_init(struct mptcp_sock *msk)
+ void __init mptcp_pm_init(void)
- int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc)
+ {
- {
+     mptcp_pm_nl_init();
--    struct mptcp_addr_info skc_local;
++    mptcp_userspace_pm_init();
-+    struct mptcp_pm_addr_entry skc_local = { 0 };
+ }
      struct mptcp_addr_info msk_local;
      if (WARN_ON_ONCE(!msk))
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc)
       * addr
       */
      mptcp_local_address((struct sock_common *)msk, &msk_local);
 -    mptcp_local_address((struct sock_common *)skc, &skc_local);
 -    if (mptcp_addresses_equal(&msk_local, &skc_local, false))
 +    mptcp_local_address((struct sock_common *)skc, &skc_local.addr);
 +    if (mptcp_addresses_equal(&msk_local, &skc_local.addr, false))
          return 0;
 +    skc_local.addr.id = 0;
 +    skc_local.flags = MPTCP_PM_ADDR_FLAG_IMPLICIT;
 +
      if (mptcp_pm_is_userspace(msk))
          return mptcp_userspace_pm_get_local_id(msk, &skc_local);
      return mptcp_pm_nl_get_local_id(msk, &skc_local);
 diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/mptcp/pm_netlink.c
 +++ b/net/mptcp/pm_netlink.c
@@ -XXX,XX +XXX,XX @@ static int mptcp_pm_nl_create_listen_socket(struct sock *sk,
      return err;
  }
 -int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc)
 +int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk,
 +                 struct mptcp_pm_addr_entry *skc)
  {
      struct mptcp_pm_addr_entry *entry;
      struct pm_nl_pernet *pernet;
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc
      pernet = pm_nl_get_pernet_from_msk(msk);
      rcu_read_lock();
 -    entry = __lookup_addr(pernet, skc);
 +    entry = __lookup_addr(pernet, &skc->addr);
      ret = entry ? entry->addr.id : -1;
      rcu_read_unlock();
      if (ret >= 0)
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc
      if (!entry)
          return -ENOMEM;
 -    entry->addr = *skc;
 -    entry->addr.id = 0;
 +    *entry = *skc;
      entry->addr.port = 0;
 -    entry->ifindex = 0;
 -    entry->flags = MPTCP_PM_ADDR_FLAG_IMPLICIT;
 -    entry->lsk = NULL;
      ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true);
      if (ret < 0)
          kfree(entry);
 diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/mptcp/pm_userspace.c
 +++ b/net/mptcp/pm_userspace.c
 @@ -XXX,XX +XXX,XX @@ mptcp_userspace_pm_lookup_addr_by_id(struct mptcp_sock *msk, unsigned int id)
- }
+     return NULL;
+ }
- int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
 -int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
 -                    struct mptcp_addr_info *skc)
-+                    struct mptcp_pm_addr_entry *skc)
++static int userspace_pm_get_local_id(struct mptcp_sock *msk,
 +                     struct mptcp_pm_addr_entry *skc)
  {
 -    struct mptcp_pm_addr_entry *entry = NULL, new_entry;
-     __be16 msk_sport =  ((struct inet_sock *)
+-    __be16 msk_sport =  ((struct inet_sock *)
-                  inet_sk((struct sock *)msk))->inet_sport;
+-                 inet_sk((struct sock *)msk))->inet_sport;
 +    struct mptcp_pm_addr_entry *entry;
      spin_lock_bh(&msk->pm.lock);
 -    entry = mptcp_userspace_pm_lookup_addr(msk, skc);
 +    entry = mptcp_userspace_pm_lookup_addr(msk, &skc->addr);
      spin_unlock_bh(&msk->pm.lock);
      if (entry)
          return entry->addr.id;
--    memset(&new_entry, 0, sizeof(struct mptcp_pm_addr_entry));
++    return mptcp_userspace_pm_append_new_local_addr(msk, skc, true);
--    new_entry.addr = *skc;
++}
--    new_entry.addr.id = 0;
++
--    new_entry.flags = MPTCP_PM_ADDR_FLAG_IMPLICIT;
++int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
--
++                    struct mptcp_addr_info *skc)
--    if (new_entry.addr.port == msk_sport)
++{
--        new_entry.addr.port = 0;
++    __be16 msk_sport = ((struct inet_sock *)
-+    if (skc->addr.port == msk_sport)
++                inet_sk((struct sock *)msk))->inet_sport;
-+        skc->addr.port = 0;
++    struct mptcp_pm_addr_entry new_entry;
 +
      memset(&new_entry, 0, sizeof(struct mptcp_pm_addr_entry));
      new_entry.addr = *skc;
      new_entry.addr.id = 0;
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
      if (new_entry.addr.port == msk_sport)
          new_entry.addr.port = 0;
 -    return mptcp_userspace_pm_append_new_local_addr(msk, &new_entry, true);
-+    return mptcp_userspace_pm_append_new_local_addr(msk, skc, true);
++    return userspace_pm_get_local_id(msk, &new_entry);
  }
- bool mptcp_userspace_pm_is_backup(struct mptcp_sock *msk,
+-u8 mptcp_userspace_pm_get_flags(struct mptcp_sock *msk,
 -                struct mptcp_addr_info *skc)
 +static u8 userspace_pm_get_flags(struct mptcp_sock *msk,
 +                 struct mptcp_addr_info *skc)
  {
      struct mptcp_pm_addr_entry *entry;
      u8 flags;
@@ -XXX,XX +XXX,XX @@ u8 mptcp_userspace_pm_get_flags(struct mptcp_sock *msk,
      return flags;
  }
 +u8 mptcp_userspace_pm_get_flags(struct mptcp_sock *msk,
 +                struct mptcp_addr_info *skc)
 +{
 +    return userspace_pm_get_flags(msk, skc);
 +}
 +
  static struct mptcp_sock *mptcp_userspace_pm_get_sock(const struct genl_info *info)
  {
      struct mptcp_sock *msk;
@@ -XXX,XX +XXX,XX @@ static struct mptcp_sock *mptcp_userspace_pm_get_sock(const struct genl_info *in
      return msk;
  }
 +static int userspace_pm_address_announce(struct mptcp_sock *msk,
 +                     struct mptcp_pm_addr_entry *local)
 +{
 +    int err;
 +
 +    err = mptcp_userspace_pm_append_new_local_addr(msk, local, false);
 +    if (err < 0)
 +        return err;
 +
 +    spin_lock_bh(&msk->pm.lock);
 +
 +    if (mptcp_pm_alloc_anno_list(msk, &local->addr)) {
 +        msk->pm.add_addr_signaled++;
 +        mptcp_pm_announce_addr(msk, &local->addr, false);
 +        mptcp_pm_nl_addr_send_ack(msk);
 +    }
 +
 +    spin_unlock_bh(&msk->pm.lock);
 +
 +    return 0;
 +}
 +
  int mptcp_pm_nl_announce_doit(struct sk_buff *skb, struct genl_info *info)
  {
      struct mptcp_pm_addr_entry addr_val;
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_announce_doit(struct sk_buff *skb, struct genl_info *info)
          goto announce_err;
      }
 -    err = mptcp_userspace_pm_append_new_local_addr(msk, &addr_val, false);
 -    if (err < 0) {
 -        NL_SET_ERR_MSG_ATTR(info->extack, addr,
 -                    "did not match address and id");
 -        goto announce_err;
 -    }
 -
      lock_sock(sk);
 -    spin_lock_bh(&msk->pm.lock);
 -
 -    if (mptcp_pm_alloc_anno_list(msk, &addr_val.addr)) {
 -        msk->pm.add_addr_signaled++;
 -        mptcp_pm_announce_addr(msk, &addr_val.addr, false);
 -        mptcp_pm_nl_addr_send_ack(msk);
 -    }
 -
 -    spin_unlock_bh(&msk->pm.lock);
 +    err = userspace_pm_address_announce(msk, &addr_val);
      release_sock(sk);
 +    if (err)
 +        NL_SET_ERR_MSG_ATTR(info->extack, addr,
 +                    "did not match address and id");
 -    err = 0;
   announce_err:
      sock_put(sk);
      return err;
@@ -XXX,XX +XXX,XX @@ void mptcp_pm_remove_addr_entry(struct mptcp_sock *msk,
      spin_unlock_bh(&msk->pm.lock);
  }
 +static int userspace_pm_address_remove(struct mptcp_sock *msk, u8 id)
 +{
 +    struct mptcp_pm_addr_entry *entry;
 +
 +    if (id == 0)
 +        return mptcp_userspace_pm_remove_id_zero_address(msk);
 +
 +    spin_lock_bh(&msk->pm.lock);
 +    entry = mptcp_userspace_pm_lookup_addr_by_id(msk, id);
 +    if (!entry) {
 +        spin_unlock_bh(&msk->pm.lock);
 +        return -EINVAL;
 +    }
 +
 +    list_del_rcu(&entry->list);
 +    spin_unlock_bh(&msk->pm.lock);
 +
 +    mptcp_pm_remove_addr_entry(msk, entry);
 +
 +    sock_kfree_s((struct sock *)msk, entry, sizeof(*entry));
 +
 +    return 0;
 +}
 +
  int mptcp_pm_nl_remove_doit(struct sk_buff *skb, struct genl_info *info)
  {
 -    struct mptcp_pm_addr_entry *match;
      struct mptcp_sock *msk;
      struct nlattr *id;
      int err = -EINVAL;
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_remove_doit(struct sk_buff *skb, struct genl_info *info)
      sk = (struct sock *)msk;
      lock_sock(sk);
 -
 -    if (id_val == 0) {
 -        err = mptcp_userspace_pm_remove_id_zero_address(msk);
 -        release_sock(sk);
 -        goto out;
 -    }
 -
 -    spin_lock_bh(&msk->pm.lock);
 -    match = mptcp_userspace_pm_lookup_addr_by_id(msk, id_val);
 -    if (!match) {
 -        spin_unlock_bh(&msk->pm.lock);
 -        release_sock(sk);
 -        goto out;
 -    }
 -
 -    list_del_rcu(&match->list);
 -    spin_unlock_bh(&msk->pm.lock);
 -
 -    mptcp_pm_remove_addr_entry(msk, match);
 -
 +    err = userspace_pm_address_remove(msk, id_val);
      release_sock(sk);
 -
 -    sock_kfree_s(sk, match, sizeof(*match));
 -
 -    err = 0;
 -out:
      if (err)
          NL_SET_ERR_MSG_ATTR_FMT(info->extack, id,
                      "address with id %u not found",
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_remove_doit(struct sk_buff *skb, struct genl_info *info)
      return err;
  }
 +static int userspace_pm_subflow_create(struct mptcp_sock *msk,
 +                       struct mptcp_pm_addr_entry *entry,
 +                       struct mptcp_addr_info *remote)
 +{
 +    struct sock *sk = (struct sock *)msk;
 +    struct mptcp_pm_local local;
 +    int err;
 +
 +    err = mptcp_userspace_pm_append_new_local_addr(msk, entry, false);
 +    if (err < 0)
 +        return err;
 +
 +    local.addr = entry->addr;
 +    local.flags = entry->flags;
 +    local.ifindex = entry->ifindex;
 +
 +    err = __mptcp_subflow_connect(sk, &local, remote);
 +    spin_lock_bh(&msk->pm.lock);
 +    if (err)
 +        mptcp_userspace_pm_delete_local_addr(msk, entry);
 +    else
 +        msk->pm.subflows++;
 +    spin_unlock_bh(&msk->pm.lock);
 +
 +    return err;
 +}
 +
  int mptcp_pm_nl_subflow_create_doit(struct sk_buff *skb, struct genl_info *info)
  {
      struct mptcp_pm_addr_entry entry = { 0 };
      struct mptcp_addr_info addr_r;
      struct nlattr *raddr, *laddr;
 -    struct mptcp_pm_local local;
      struct mptcp_sock *msk;
      int err = -EINVAL;
      struct sock *sk;
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_subflow_create_doit(struct sk_buff *skb, struct genl_info *info)
          goto create_err;
      }
 -    err = mptcp_userspace_pm_append_new_local_addr(msk, &entry, false);
 -    if (err < 0) {
 -        NL_SET_ERR_MSG_ATTR(info->extack, laddr,
 -                    "did not match address and id");
 -        goto create_err;
 -    }
 -
 -    local.addr = entry.addr;
 -    local.flags = entry.flags;
 -    local.ifindex = entry.ifindex;
 -
      lock_sock(sk);
 -    err = __mptcp_subflow_connect(sk, &local, &addr_r);
 +    err = userspace_pm_subflow_create(msk, &entry, &addr_r);
      release_sock(sk);
      if (err)
          GENL_SET_ERR_MSG_FMT(info, "connect error: %d", err);
 -    spin_lock_bh(&msk->pm.lock);
 -    if (err)
 -        mptcp_userspace_pm_delete_local_addr(msk, &entry);
 -    else
 -        msk->pm.subflows++;
 -    spin_unlock_bh(&msk->pm.lock);
 -
   create_err:
      sock_put(sk);
      return err;
@@ -XXX,XX +XXX,XX @@ static struct sock *mptcp_nl_find_ssk(struct mptcp_sock *msk,
      return NULL;
  }
 +static int userspace_pm_subflow_destroy(struct mptcp_sock *msk,
 +                    struct mptcp_pm_addr_entry *local,
 +                    struct mptcp_addr_info *remote)
 +{
 +    struct sock *ssk, *sk = (struct sock *)msk;
 +
 +    ssk = mptcp_nl_find_ssk(msk, &local->addr, remote);
 +    if (!ssk)
 +        return -ESRCH;
 +
 +    spin_lock_bh(&msk->pm.lock);
 +    mptcp_userspace_pm_delete_local_addr(msk, local);
 +    spin_unlock_bh(&msk->pm.lock);
 +    mptcp_subflow_shutdown(sk, ssk, RCV_SHUTDOWN | SEND_SHUTDOWN);
 +    mptcp_close_ssk(sk, ssk, mptcp_subflow_ctx(ssk));
 +    MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RMSUBFLOW);
 +
 +    return 0;
 +}
 +
  int mptcp_pm_nl_subflow_destroy_doit(struct sk_buff *skb, struct genl_info *info)
  {
      struct mptcp_pm_addr_entry addr_l;
      struct mptcp_addr_info addr_r;
      struct nlattr *raddr, *laddr;
      struct mptcp_sock *msk;
 -    struct sock *sk, *ssk;
      int err = -EINVAL;
 +    struct sock *sk;
      if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ATTR_ADDR) ||
          GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ATTR_ADDR_REMOTE))
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_subflow_destroy_doit(struct sk_buff *skb, struct genl_info *info
      }
      lock_sock(sk);
 -    ssk = mptcp_nl_find_ssk(msk, &addr_l.addr, &addr_r);
 -    if (!ssk) {
 -        GENL_SET_ERR_MSG(info, "subflow not found");
 -        err = -ESRCH;
 -        goto release_sock;
 -    }
 -
 -    spin_lock_bh(&msk->pm.lock);
 -    mptcp_userspace_pm_delete_local_addr(msk, &addr_l);
 -    spin_unlock_bh(&msk->pm.lock);
 -    mptcp_subflow_shutdown(sk, ssk, RCV_SHUTDOWN | SEND_SHUTDOWN);
 -    mptcp_close_ssk(sk, ssk, mptcp_subflow_ctx(ssk));
 -    MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RMSUBFLOW);
 -release_sock:
 +    err = userspace_pm_subflow_destroy(msk, &addr_l, &addr_r);
      release_sock(sk);
 +    if (err)
 +        GENL_SET_ERR_MSG(info, "subflow not found");
  destroy_err:
      sock_put(sk);
      return err;
  }
 +static int userspace_pm_set_flags(struct mptcp_sock *msk,
 +                  struct mptcp_pm_addr_entry *local,
 +                  struct mptcp_addr_info *remote)
 +{
 +    struct mptcp_pm_addr_entry *entry;
 +    u8 bkup = 0;
 +
 +    if (local->flags & MPTCP_PM_ADDR_FLAG_BACKUP)
 +        bkup = 1;
 +
 +    spin_lock_bh(&msk->pm.lock);
 +    entry = mptcp_userspace_pm_lookup_addr(msk, &local->addr);
 +    if (entry) {
 +        if (bkup)
 +            entry->flags |= MPTCP_PM_ADDR_FLAG_BACKUP;
 +        else
 +            entry->flags &= ~MPTCP_PM_ADDR_FLAG_BACKUP;
 +    }
 +    spin_unlock_bh(&msk->pm.lock);
 +
 +    return mptcp_pm_nl_mp_prio_send_ack(msk, &local->addr, remote, bkup);
 +}
 +
  int mptcp_userspace_pm_set_flags(struct mptcp_pm_addr_entry *local,
                   struct genl_info *info)
  {
      struct mptcp_addr_info rem = { .family = AF_UNSPEC, };
 -    struct mptcp_pm_addr_entry *entry;
      struct nlattr *attr, *attr_rem;
      struct mptcp_sock *msk;
      int ret = -EINVAL;
      struct sock *sk;
 -    u8 bkup = 0;
      if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ATTR_ADDR_REMOTE))
          return ret;
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_set_flags(struct mptcp_pm_addr_entry *local,
          goto set_flags_err;
      }
 -    if (local->flags & MPTCP_PM_ADDR_FLAG_BACKUP)
 -        bkup = 1;
 -
 -    spin_lock_bh(&msk->pm.lock);
 -    entry = mptcp_userspace_pm_lookup_addr(msk, &local->addr);
 -    if (entry) {
 -        if (bkup)
 -            entry->flags |= MPTCP_PM_ADDR_FLAG_BACKUP;
 -        else
 -            entry->flags &= ~MPTCP_PM_ADDR_FLAG_BACKUP;
 -    }
 -    spin_unlock_bh(&msk->pm.lock);
 -
      lock_sock(sk);
 -    ret = mptcp_pm_nl_mp_prio_send_ack(msk, &local->addr, &rem, bkup);
 +    ret = userspace_pm_set_flags(msk, local, &rem);
      release_sock(sk);
      /* mptcp_pm_nl_mp_prio_send_ack() only fails in one case */
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_get_addr(u8 id, struct mptcp_pm_addr_entry *addr,
      return ret;
  }
 +static struct mptcp_pm_ops mptcp_userspace_pm = {
 +    .address_announce    = userspace_pm_address_announce,
 +    .address_remove        = userspace_pm_address_remove,
 +    .subflow_create        = userspace_pm_subflow_create,
 +    .subflow_destroy    = userspace_pm_subflow_destroy,
 +    .get_local_id        = userspace_pm_get_local_id,
 +    .get_flags        = userspace_pm_get_flags,
 +    .set_flags        = userspace_pm_set_flags,
 +    .type            = MPTCP_PM_TYPE_USERSPACE,
 +    .owner            = THIS_MODULE,
 +};
 +
  /* Must be called with rcu read lock held */
  struct mptcp_pm_ops *mptcp_pm_find(enum mptcp_pm_type type)
  {
@@ -XXX,XX +XXX,XX @@ int mptcp_register_path_manager(struct mptcp_pm_ops *pm)
  void mptcp_unregister_path_manager(struct mptcp_pm_ops *pm)
  {
 +    if (pm == &mptcp_userspace_pm)
 +        return;
 +
      spin_lock(&mptcp_pm_list_lock);
      list_del_rcu(&pm->list);
      spin_unlock(&mptcp_pm_list_lock);
  }
 +
 +void __init mptcp_userspace_pm_init(void)
 +{
 +    mptcp_register_path_manager(&mptcp_userspace_pm);
 +}
 diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
 index XXXXXXX..XXXXXXX 100644
 --- a/net/mptcp/protocol.h
 +++ b/net/mptcp/protocol.h
-@@ -XXX,XX +XXX,XX @@ bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
+@@ -XXX,XX +XXX,XX @@ static inline u8 subflow_get_local_id(const struct mptcp_subflow_context *subflo
- bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
+ }
-                  struct mptcp_rm_list *rm_list);
- int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc);
+ void __init mptcp_pm_nl_init(void);
--int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc);
++void __init mptcp_userspace_pm_init(void);
--int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc);
+ void mptcp_pm_nl_work(struct mptcp_sock *msk);
-+int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk,
+ unsigned int mptcp_pm_get_add_addr_signal_max(const struct mptcp_sock *msk);
-+                 struct mptcp_pm_addr_entry *skc);
+ unsigned int mptcp_pm_get_add_addr_accept_max(const struct mptcp_sock *msk);
 +int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
 +                    struct mptcp_pm_addr_entry *skc);
  bool mptcp_pm_is_backup(struct mptcp_sock *msk, struct sock_common *skc);
  bool mptcp_pm_nl_is_backup(struct mptcp_sock *msk, struct mptcp_addr_info *skc);
  bool mptcp_userspace_pm_is_backup(struct mptcp_sock *msk, struct mptcp_addr_info *skc);
 --
 .43.0

-[PATCH mptcp-next v1 4/6] mptcp: pm: in-kernel: use kmemdup helper
+[PATCH mptcp-next v3 3/3] mptcp: init and release mptcp_pm_ops
 From: Geliang Tang <tanggeliang@kylinos.cn>
-Instead of using kmalloc() or kzalloc() to allocate an entry and
+Add a struct mptcp_pm_ops pointer "ops" in struct mptcp_pm_data, and two
-then immediately duplicate another entry to the newly allocated
+functions mptcp_init_pm() and mptcp_release_pm(), to set and release this
-one, kmemdup() helper can be used to simplify the code.
+pointer. mptcp_init_pm() is invoked in mptcp_pm_data_reset(), while
 mptcp_release_pm() is invoked in __mptcp_destroy_sock().
 In this way, different userspace path managers can be initialized through
 the pm_type sysctl, and then called into their respective interfaces
 through "ops" of "msk->pm".
 Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
 ---
- net/mptcp/pm_netlink.c | 6 ++----
+ net/mptcp/pm.c           |  4 +++
-file changed, 2 insertions(+), 4 deletions(-)
+ net/mptcp/pm_userspace.c | 58 +++++++++++++++++++++++++++++++++++-----
  net/mptcp/protocol.c     |  1 +
  net/mptcp/protocol.h     |  3 +++
 files changed, 59 insertions(+), 7 deletions(-)
-diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
+diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
 index XXXXXXX..XXXXXXX 100644
---- a/net/mptcp/pm_netlink.c
+--- a/net/mptcp/pm.c
-+++ b/net/mptcp/pm_netlink.c
++++ b/net/mptcp/pm.c
-@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk,
+@@ -XXX,XX +XXX,XX @@ void mptcp_pm_data_reset(struct mptcp_sock *msk)
-         return ret;
+         WRITE_ONCE(pm->work_pending, 0);
+         WRITE_ONCE(pm->accept_addr, 0);
-     /* address not found, add to local list */
+         WRITE_ONCE(pm->accept_subflow, 0);
--    entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
++
-+    entry = kmemdup(skc, sizeof(*skc), GFP_ATOMIC);
++        rcu_read_lock();
-     if (!entry)
++        mptcp_init_pm(msk, mptcp_pm_find(pm_type));
-         return -ENOMEM;
++        rcu_read_unlock();
 -    *entry = *skc;
      entry->addr.port = 0;
      ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true);
      if (ret < 0)
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info)
          return -EINVAL;
      }
--    entry = kzalloc(sizeof(*entry), GFP_KERNEL_ACCOUNT);
+     WRITE_ONCE(pm->addr_signal, 0);
-+    entry = kmemdup(&addr, sizeof(addr), GFP_KERNEL_ACCOUNT);
+diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c
-     if (!entry) {
+index XXXXXXX..XXXXXXX 100644
-         GENL_SET_ERR_MSG(info, "can't allocate addr");
+--- a/net/mptcp/pm_userspace.c
-         return -ENOMEM;
++++ b/net/mptcp/pm_userspace.c
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
      if (new_entry.addr.port == msk_sport)
          new_entry.addr.port = 0;
 -    return userspace_pm_get_local_id(msk, &new_entry);
 +    return msk->pm.ops->get_local_id ?
 +           msk->pm.ops->get_local_id(msk, &new_entry) :
 +           userspace_pm_get_local_id(msk, &new_entry);
  }
  static u8 userspace_pm_get_flags(struct mptcp_sock *msk,
@@ -XXX,XX +XXX,XX @@ static u8 userspace_pm_get_flags(struct mptcp_sock *msk,
  u8 mptcp_userspace_pm_get_flags(struct mptcp_sock *msk,
                  struct mptcp_addr_info *skc)
  {
 -    return userspace_pm_get_flags(msk, skc);
 +    return msk->pm.ops->get_flags ?
 +           msk->pm.ops->get_flags(msk, skc) :
 +           userspace_pm_get_flags(msk, skc);
  }
  static struct mptcp_sock *mptcp_userspace_pm_get_sock(const struct genl_info *info)
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_announce_doit(struct sk_buff *skb, struct genl_info *info)
      }
--    *entry = addr;
+     lock_sock(sk);
-     if (entry->addr.port) {
+-    err = userspace_pm_address_announce(msk, &addr_val);
-         ret = mptcp_pm_nl_create_listen_socket(skb->sk, entry);
++    err = msk->pm.ops->address_announce ?
-         if (ret) {
++          msk->pm.ops->address_announce(msk, &addr_val) :
 +          userspace_pm_address_announce(msk, &addr_val);
      release_sock(sk);
      if (err)
          NL_SET_ERR_MSG_ATTR(info->extack, addr,
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_remove_doit(struct sk_buff *skb, struct genl_info *info)
      sk = (struct sock *)msk;
      lock_sock(sk);
 -    err = userspace_pm_address_remove(msk, id_val);
 +    err = msk->pm.ops->address_remove ?
 +          msk->pm.ops->address_remove(msk, id_val) :
 +          userspace_pm_address_remove(msk, id_val);
      release_sock(sk);
      if (err)
          NL_SET_ERR_MSG_ATTR_FMT(info->extack, id,
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_subflow_create_doit(struct sk_buff *skb, struct genl_info *info)
      }
      lock_sock(sk);
 -    err = userspace_pm_subflow_create(msk, &entry, &addr_r);
 +    err = msk->pm.ops->subflow_create ?
 +          msk->pm.ops->subflow_create(msk, &entry, &addr_r) :
 +          userspace_pm_subflow_create(msk, &entry, &addr_r);
      release_sock(sk);
      if (err)
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_subflow_destroy_doit(struct sk_buff *skb, struct genl_info *info
      }
      lock_sock(sk);
 -    err = userspace_pm_subflow_destroy(msk, &addr_l, &addr_r);
 +    err = msk->pm.ops->subflow_destroy ?
 +          msk->pm.ops->subflow_destroy(msk, &addr_l, &addr_r) :
 +          userspace_pm_subflow_destroy(msk, &addr_l, &addr_r);
      release_sock(sk);
      if (err)
          GENL_SET_ERR_MSG(info, "subflow not found");
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_set_flags(struct mptcp_pm_addr_entry *local,
      }
      lock_sock(sk);
 -    ret = userspace_pm_set_flags(msk, local, &rem);
 +    ret = msk->pm.ops->set_flags ?
 +          msk->pm.ops->set_flags(msk, local, &rem) :
 +          userspace_pm_set_flags(msk, local, &rem);
      release_sock(sk);
      /* mptcp_pm_nl_mp_prio_send_ack() only fails in one case */
@@ -XXX,XX +XXX,XX @@ void mptcp_unregister_path_manager(struct mptcp_pm_ops *pm)
      spin_unlock(&mptcp_pm_list_lock);
  }
 +int mptcp_init_pm(struct mptcp_sock *msk, struct mptcp_pm_ops *pm)
 +{
 +    if (!pm)
 +        pm = &mptcp_userspace_pm;
 +
 +    if (!bpf_try_module_get(pm, pm->owner))
 +        return -EBUSY;
 +
 +    msk->pm.ops = pm;
 +    if (msk->pm.ops->init)
 +        msk->pm.ops->init(msk);
 +
 +    pr_debug("userspace_pm type %u initialized\n", msk->pm.ops->type);
 +    return 0;
 +}
 +
 +void mptcp_release_pm(struct mptcp_sock *msk)
 +{
 +    struct mptcp_pm_ops *pm = msk->pm.ops;
 +
 +    if (!pm)
 +        return;
 +
 +    msk->pm.ops = NULL;
 +    if (pm->release)
 +        pm->release(msk);
 +
 +    bpf_module_put(pm, pm->owner);
 +}
 +
  void __init mptcp_userspace_pm_init(void)
  {
      mptcp_register_path_manager(&mptcp_userspace_pm);
 diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/mptcp/protocol.c
 +++ b/net/mptcp/protocol.c
@@ -XXX,XX +XXX,XX @@ static void __mptcp_destroy_sock(struct sock *sk)
      sk_stop_timer(sk, &sk->sk_timer);
      msk->pm.status = 0;
      mptcp_release_sched(msk);
 +    mptcp_release_pm(msk);
      sk->sk_prot->destroy(sk);
 diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
 index XXXXXXX..XXXXXXX 100644
 --- a/net/mptcp/protocol.h
 +++ b/net/mptcp/protocol.h
@@ -XXX,XX +XXX,XX @@ struct mptcp_pm_data {
      struct mptcp_addr_info remote;
      struct list_head anno_list;
      struct list_head userspace_pm_local_addr_list;
 +    struct mptcp_pm_ops *ops;
      spinlock_t    lock;        /*protects the whole PM data */
@@ -XXX,XX +XXX,XX @@ struct mptcp_pm_ops *mptcp_pm_find(enum mptcp_pm_type type);
  int mptcp_validate_path_manager(struct mptcp_pm_ops *pm);
  int mptcp_register_path_manager(struct mptcp_pm_ops *pm);
  void mptcp_unregister_path_manager(struct mptcp_pm_ops *pm);
 +int mptcp_init_pm(struct mptcp_sock *msk, struct mptcp_pm_ops *pm);
 +void mptcp_release_pm(struct mptcp_sock *msk);
  void mptcp_free_local_addr_list(struct mptcp_sock *msk);
 --
 .43.0

-[PATCH mptcp-next v1 5/6] sock: add sock_kmemdup helper
+Deleted patch
-From: Geliang Tang <tanggeliang@kylinos.cn>
-This patch adds the sock version of kmemdup() helper, named sock_kmemdup(),
-to duplicate a memory block using the socket's option memory buffer.
-Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
----
- include/net/sock.h |  1 +
- net/core/sock.c    | 23 +++++++++++++++++++++++
-files changed, 24 insertions(+)
-diff --git a/include/net/sock.h b/include/net/sock.h
-index XXXXXXX..XXXXXXX 100644
---- a/include/net/sock.h
-+++ b/include/net/sock.h
-@@ -XXX,XX +XXX,XX @@ static inline struct sk_buff *sock_alloc_send_skb(struct sock *sk,
- }
- void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
-+void *sock_kmemdup(struct sock *sk, const void *src, int size, gfp_t priority);
- void sock_kfree_s(struct sock *sk, void *mem, int size);
- void sock_kzfree_s(struct sock *sk, void *mem, int size);
- void sk_send_sigurg(struct sock *sk);
-diff --git a/net/core/sock.c b/net/core/sock.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/core/sock.c
-+++ b/net/core/sock.c
-@@ -XXX,XX +XXX,XX @@ void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
- }
- EXPORT_SYMBOL(sock_kmalloc);
-+/*
-+ * Duplicate a memory block using the socket's option memory buffer.
-+ */
-+void *sock_kmemdup(struct sock *sk, const void *src, int size, gfp_t priority)
-+{
-+    int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
-+
-+    if ((unsigned int)size <= optmem_max &&
-+        atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
-+        void *mem;
-+        /* First do the add, to avoid the race if kmalloc
-+         * might sleep.
-+         */
-+        atomic_add(size, &sk->sk_omem_alloc);
-+        mem = kmemdup(src, size, priority);
-+        if (mem)
-+            return mem;
-+        atomic_sub(size, &sk->sk_omem_alloc);
-+    }
-+    return NULL;
-+}
-+EXPORT_SYMBOL(sock_kmemdup);
-+
- /* Free an option memory block. Note, we actually want the inline
-  * here as this allows gcc to detect the nullify and fold away the
-  * condition entirely.
---
-.43.0

From: Geliang Tang <tanggeliang@kylinos.cn>

The number of parameters in mptcp_nl_set_flags() can be reduced.
Only need to pass a "local" parameter to it instead of "local->addr"
and "local->flags".

Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
---
 net/mptcp/pm_netlink.c | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm_netlink.c
+++ b/net/mptcp/pm_netlink.c
@@ -XXX,XX +XXX,XX @@ static void mptcp_pm_nl_fullmesh(struct mptcp_sock *msk,
 	spin_unlock_bh(&msk->pm.lock);
 }
 
-static void mptcp_nl_set_flags(struct net *net, struct mptcp_addr_info *addr,
-			       u8 flags, u8 changed)
+static void mptcp_nl_set_flags(struct net *net,
+			       struct mptcp_pm_addr_entry *local,
+			       u8 changed)
 {
-	u8 is_subflow = !!(flags & MPTCP_PM_ADDR_FLAG_SUBFLOW);
-	u8 bkup = !!(flags & MPTCP_PM_ADDR_FLAG_BACKUP);
+	u8 is_subflow = !!(local->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW);
+	u8 bkup = !!(local->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
 	long s_slot = 0, s_num = 0;
 	struct mptcp_sock *msk;
 
@@ -XXX,XX +XXX,XX @@ static void mptcp_nl_set_flags(struct net *net, struct mptcp_addr_info *addr,
 
 		lock_sock(sk);
 		if (changed & MPTCP_PM_ADDR_FLAG_BACKUP)
-			mptcp_pm_nl_mp_prio_send_ack(msk, addr, NULL, bkup);
+			mptcp_pm_nl_mp_prio_send_ack(msk, &local->addr, NULL, bkup);
 		/* Subflows will only be recreated if the SUBFLOW flag is set */
 		if (is_subflow && (changed & MPTCP_PM_ADDR_FLAG_FULLMESH))
-			mptcp_pm_nl_fullmesh(msk, addr);
+			mptcp_pm_nl_fullmesh(msk, &local->addr);
 		release_sock(sk);
 
 next:
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_set_flags(struct mptcp_pm_addr_entry *local,
 	*local = *entry;
 	spin_unlock_bh(&pernet->lock);
 
-	mptcp_nl_set_flags(net, &local->addr, local->flags, changed);
+	mptcp_nl_set_flags(net, local, changed);
 	return 0;
 }
 
-- 
2.43.0

From: Geliang Tang <tanggeliang@kylinos.cn>

The following code in mptcp_userspace_pm_get_local_id() that assigns "skc"
to "new_entry" is not allowed in BPF if we use the same code to implement
the get_local_id() interface of a BFP path manager:

memset(&new_entry, 0, sizeof(struct mptcp_pm_addr_entry));
	new_entry.addr = *skc;
	new_entry.addr.id = 0;
	new_entry.flags = MPTCP_PM_ADDR_FLAG_IMPLICIT;

To solve the issue, this patch moves this assignment to "new_entry" forward
to mptcp_pm_get_local_id(), and then passing "new_entry" as a parameter to
both mptcp_pm_nl_get_local_id() and mptcp_userspace_pm_get_local_id().

Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
---
 net/mptcp/pm.c           |  9 ++++++---
 net/mptcp/pm_netlink.c   | 11 ++++-------
 net/mptcp/pm_userspace.c | 17 ++++++-----------
 net/mptcp/protocol.h     |  6 ++++--
 4 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm.c
+++ b/net/mptcp/pm.c
@@ -XXX,XX +XXX,XX @@ bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
 
 int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc)
 {
-	struct mptcp_addr_info skc_local;
+	struct mptcp_pm_addr_entry skc_local = { 0 };
 	struct mptcp_addr_info msk_local;
 
 	if (WARN_ON_ONCE(!msk))
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc)
 	 * addr
 	 */
 	mptcp_local_address((struct sock_common *)msk, &msk_local);
-	mptcp_local_address((struct sock_common *)skc, &skc_local);
-	if (mptcp_addresses_equal(&msk_local, &skc_local, false))
+	mptcp_local_address((struct sock_common *)skc, &skc_local.addr);
+	if (mptcp_addresses_equal(&msk_local, &skc_local.addr, false))
 		return 0;
 
+	skc_local.addr.id = 0;
+	skc_local.flags = MPTCP_PM_ADDR_FLAG_IMPLICIT;
+
 	if (mptcp_pm_is_userspace(msk))
 		return mptcp_userspace_pm_get_local_id(msk, &skc_local);
 	return mptcp_pm_nl_get_local_id(msk, &skc_local);
diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm_netlink.c
+++ b/net/mptcp/pm_netlink.c
@@ -XXX,XX +XXX,XX @@ static int mptcp_pm_nl_create_listen_socket(struct sock *sk,
 	return err;
 }
 
-int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc)
+int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk,
+			     struct mptcp_pm_addr_entry *skc)
 {
 	struct mptcp_pm_addr_entry *entry;
 	struct pm_nl_pernet *pernet;
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc
 	pernet = pm_nl_get_pernet_from_msk(msk);
 
 	rcu_read_lock();
-	entry = __lookup_addr(pernet, skc);
+	entry = __lookup_addr(pernet, &skc->addr);
 	ret = entry ? entry->addr.id : -1;
 	rcu_read_unlock();
 	if (ret >= 0)
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc
 	if (!entry)
 		return -ENOMEM;
 
-	entry->addr = *skc;
-	entry->addr.id = 0;
+	*entry = *skc;
 	entry->addr.port = 0;
-	entry->ifindex = 0;
-	entry->flags = MPTCP_PM_ADDR_FLAG_IMPLICIT;
-	entry->lsk = NULL;
 	ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true);
 	if (ret < 0)
 		kfree(entry);
diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm_userspace.c
+++ b/net/mptcp/pm_userspace.c
@@ -XXX,XX +XXX,XX @@ mptcp_userspace_pm_lookup_addr_by_id(struct mptcp_sock *msk, unsigned int id)
 }
 
 int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
-				    struct mptcp_addr_info *skc)
+				    struct mptcp_pm_addr_entry *skc)
 {
-	struct mptcp_pm_addr_entry *entry = NULL, new_entry;
 	__be16 msk_sport =  ((struct inet_sock *)
 			     inet_sk((struct sock *)msk))->inet_sport;
+	struct mptcp_pm_addr_entry *entry;
 
 	spin_lock_bh(&msk->pm.lock);
-	entry = mptcp_userspace_pm_lookup_addr(msk, skc);
+	entry = mptcp_userspace_pm_lookup_addr(msk, &skc->addr);
 	spin_unlock_bh(&msk->pm.lock);
 	if (entry)
 		return entry->addr.id;
 
-	memset(&new_entry, 0, sizeof(struct mptcp_pm_addr_entry));
-	new_entry.addr = *skc;
-	new_entry.addr.id = 0;
-	new_entry.flags = MPTCP_PM_ADDR_FLAG_IMPLICIT;
-
-	if (new_entry.addr.port == msk_sport)
-		new_entry.addr.port = 0;
+	if (skc->addr.port == msk_sport)
+		skc->addr.port = 0;
 
-	return mptcp_userspace_pm_append_new_local_addr(msk, &new_entry, true);
+	return mptcp_userspace_pm_append_new_local_addr(msk, skc, true);
 }
 
 bool mptcp_userspace_pm_is_backup(struct mptcp_sock *msk,
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -XXX,XX +XXX,XX @@ bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
 bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
 			     struct mptcp_rm_list *rm_list);
 int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc);
-int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc);
-int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc);
+int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk,
+			     struct mptcp_pm_addr_entry *skc);
+int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
+				    struct mptcp_pm_addr_entry *skc);
 bool mptcp_pm_is_backup(struct mptcp_sock *msk, struct sock_common *skc);
 bool mptcp_pm_nl_is_backup(struct mptcp_sock *msk, struct mptcp_addr_info *skc);
 bool mptcp_userspace_pm_is_backup(struct mptcp_sock *msk, struct mptcp_addr_info *skc);
-- 
2.43.0

From: Geliang Tang <tanggeliang@kylinos.cn>

Instead of using kmalloc() or kzalloc() to allocate an entry and
then immediately duplicate another entry to the newly allocated
one, kmemdup() helper can be used to simplify the code.

Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
---
 net/mptcp/pm_netlink.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm_netlink.c
+++ b/net/mptcp/pm_netlink.c
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk,
 		return ret;
 
 	/* address not found, add to local list */
-	entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
+	entry = kmemdup(skc, sizeof(*skc), GFP_ATOMIC);
 	if (!entry)
 		return -ENOMEM;
 
-	*entry = *skc;
 	entry->addr.port = 0;
 	ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true);
 	if (ret < 0)
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info)
 		return -EINVAL;
 	}
 
-	entry = kzalloc(sizeof(*entry), GFP_KERNEL_ACCOUNT);
+	entry = kmemdup(&addr, sizeof(addr), GFP_KERNEL_ACCOUNT);
 	if (!entry) {
 		GENL_SET_ERR_MSG(info, "can't allocate addr");
 		return -ENOMEM;
 	}
 
-	*entry = addr;
 	if (entry->addr.port) {
 		ret = mptcp_pm_nl_create_listen_socket(skb->sk, entry);
 		if (ret) {
-- 
2.43.0

From: Geliang Tang <tanggeliang@kylinos.cn>

This patch adds the sock version of kmemdup() helper, named sock_kmemdup(),
to duplicate a memory block using the socket's option memory buffer.

Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
---
 include/net/sock.h |  1 +
 net/core/sock.c    | 23 +++++++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/include/net/sock.h b/include/net/sock.h
index XXXXXXX..XXXXXXX 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -XXX,XX +XXX,XX @@ static inline struct sk_buff *sock_alloc_send_skb(struct sock *sk,
 }
 
 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
+void *sock_kmemdup(struct sock *sk, const void *src, int size, gfp_t priority);
 void sock_kfree_s(struct sock *sk, void *mem, int size);
 void sock_kzfree_s(struct sock *sk, void *mem, int size);
 void sk_send_sigurg(struct sock *sk);
diff --git a/net/core/sock.c b/net/core/sock.c
index XXXXXXX..XXXXXXX 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -XXX,XX +XXX,XX @@ void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
 }
 EXPORT_SYMBOL(sock_kmalloc);
 
+/*
+ * Duplicate a memory block using the socket's option memory buffer.
+ */
+void *sock_kmemdup(struct sock *sk, const void *src, int size, gfp_t priority)
+{
+	int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
+
+	if ((unsigned int)size <= optmem_max &&
+	    atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
+		void *mem;
+		/* First do the add, to avoid the race if kmalloc
+		 * might sleep.
+		 */
+		atomic_add(size, &sk->sk_omem_alloc);
+		mem = kmemdup(src, size, priority);
+		if (mem)
+			return mem;
+		atomic_sub(size, &sk->sk_omem_alloc);
+	}
+	return NULL;
+}
+EXPORT_SYMBOL(sock_kmemdup);
+
 /* Free an option memory block. Note, we actually want the inline
  * here as this allows gcc to detect the nullify and fold away the
  * condition entirely.
-- 
2.43.0

From: Geliang Tang <tanggeliang@kylinos.cn>

Instead of using sock_kmalloc() to allocate an entry and then
immediately duplicate another entry to the newly allocated one,
sock_kmemdup() helper can be used to simplify the code.

More importantly, the code "*e = *entry;" that assigns "entry"
to "e" is not easy to implemented in BPF if we use the same code
to implement an append_new_local_addr() helper of a BFP path
manager. This patch avoids this type of memory assignment
operation.

Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
---
 net/mptcp/pm_userspace.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm_userspace.c
+++ b/net/mptcp/pm_userspace.c
@@ -XXX,XX +XXX,XX @@ static int mptcp_userspace_pm_append_new_local_addr(struct mptcp_sock *msk,
 		/* Memory for the entry is allocated from the
 		 * sock option buffer.
 		 */
-		e = sock_kmalloc(sk, sizeof(*e), GFP_ATOMIC);
+		e = sock_kmemdup(sk, entry, sizeof(*entry), GFP_ATOMIC);
 		if (!e) {
 			ret = -ENOMEM;
 			goto append_err;
 		}
 
-		*e = *entry;
 		if (!e->addr.id && needs_id)
 			e->addr.id = find_next_zero_bit(id_bitmap,
 							MPTCP_PM_MAX_ADDR_ID + 1,
-- 
2.43.0

From: Geliang Tang <tanggeliang@kylinos.cn>

v3:
 - rename the 2nd parameter of get_local_id() from 'local' to 'skc'.
 - keep the 'msk_sport' check in mptcp_userspace_pm_get_local_id().
 - return 'err' instead of '0' in userspace_pm_subflow_create().
 - drop 'ret' variable inmptcp_pm_data_reset().
 - fix typos in commit log.

Depends on: "BPF path manager, part 3" v4
Based-on: <cover.1737012165.git.tanggeliang@kylinos.cn>

v2:
 - update get_local_id interface in patch 2.

get_addr() and dump_addr() interfaces of BPF userspace pm are dropped
as Matt suggested.

In order to implement BPF userspace path manager, it is necessary to
unify the interfaces of the path manager. This set contains some
cleanups and refactoring to unify the interfaces in kernel space.
Finally, define a struct mptcp_pm_ops for a userspace path manager
like this:

struct mptcp_pm_ops {
	int (*address_announce)(struct mptcp_sock *msk,
				struct mptcp_pm_addr_entry *local);
	int (*address_remove)(struct mptcp_sock *msk, u8 id);
	int (*subflow_create)(struct mptcp_sock *msk,
			      struct mptcp_pm_addr_entry *local,
			      struct mptcp_addr_info *remote);
	int (*subflow_destroy)(struct mptcp_sock *msk,
			       struct mptcp_pm_addr_entry *local,
			       struct mptcp_addr_info *remote);
	int (*get_local_id)(struct mptcp_sock *msk,
			    struct mptcp_pm_addr_entry *skc);
	u8 (*get_flags)(struct mptcp_sock *msk,
			struct mptcp_addr_info *skc);
	int (*set_flags)(struct mptcp_sock *msk,
			 struct mptcp_pm_addr_entry *local,
			 struct mptcp_addr_info *remote);

u8			type;
	struct module		*owner;
	struct list_head	list;

void (*init)(struct mptcp_sock *msk);
	void (*release)(struct mptcp_sock *msk);
} ____cacheline_aligned_in_smp;

Closes: https://github.com/multipath-tcp/mptcp_net-next/issues/74

Geliang Tang (3):
  mptcp: define struct mptcp_pm_ops
  mptcp: register default userspace pm
  mptcp: init and release mptcp_pm_ops

-- 
2.43.0

From: Geliang Tang <tanggeliang@kylinos.cn>

In order to allow users to develop their own BPF-based path manager,
this patch defines a struct ops "mptcp_pm_ops" for a userspace path
manager, which contains a set of interfaces.

Add a set of functions to register, unregister, find and validate a
given struct ops.

Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
---
 include/net/mptcp.h      | 27 +++++++++++++++++++
 net/mptcp/pm_userspace.c | 57 ++++++++++++++++++++++++++++++++++++++++
 net/mptcp/protocol.h     |  5 ++++
 3 files changed, 89 insertions(+)

diff --git a/include/net/mptcp.h b/include/net/mptcp.h
index XXXXXXX..XXXXXXX 100644
--- a/include/net/mptcp.h
+++ b/include/net/mptcp.h
@@ -XXX,XX +XXX,XX @@
 
 struct mptcp_info;
 struct mptcp_sock;
+struct mptcp_pm_addr_entry;
 struct seq_file;
 
 /* MPTCP sk_buff extension data */
@@ -XXX,XX +XXX,XX @@ struct mptcp_sched_ops {
 	void (*release)(struct mptcp_sock *msk);
 } ____cacheline_aligned_in_smp;
 
+struct mptcp_pm_ops {
+	int (*address_announce)(struct mptcp_sock *msk,
+				struct mptcp_pm_addr_entry *local);
+	int (*address_remove)(struct mptcp_sock *msk, u8 id);
+	int (*subflow_create)(struct mptcp_sock *msk,
+			      struct mptcp_pm_addr_entry *local,
+			      struct mptcp_addr_info *remote);
+	int (*subflow_destroy)(struct mptcp_sock *msk,
+			       struct mptcp_pm_addr_entry *local,
+			       struct mptcp_addr_info *remote);
+	int (*get_local_id)(struct mptcp_sock *msk,
+			    struct mptcp_pm_addr_entry *skc);
+	u8 (*get_flags)(struct mptcp_sock *msk,
+			struct mptcp_addr_info *skc);
+	int (*set_flags)(struct mptcp_sock *msk,
+			 struct mptcp_pm_addr_entry *local,
+			 struct mptcp_addr_info *remote);
+
+	u8			type;
+	struct module		*owner;
+	struct list_head	list;
+
+	void (*init)(struct mptcp_sock *msk);
+	void (*release)(struct mptcp_sock *msk);
+} ____cacheline_aligned_in_smp;
+
 #ifdef CONFIG_MPTCP
 void mptcp_init(void);
 
diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm_userspace.c
+++ b/net/mptcp/pm_userspace.c
@@ -XXX,XX +XXX,XX @@
  * Copyright (c) 2022, Intel Corporation.
  */
 
+#include <linux/rculist.h>
+#include <linux/spinlock.h>
 #include "protocol.h"
 #include "mib.h"
 #include "mptcp_pm_gen.h"
@@ -XXX,XX +XXX,XX @@
 	list_for_each_entry(__entry,						\
 			    &((__msk)->pm.userspace_pm_local_addr_list), list)
 
+static DEFINE_SPINLOCK(mptcp_pm_list_lock);
+static LIST_HEAD(mptcp_pm_list);
+
 void mptcp_free_local_addr_list(struct mptcp_sock *msk)
 {
 	struct mptcp_pm_addr_entry *entry, *tmp;
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_get_addr(u8 id, struct mptcp_pm_addr_entry *addr,
 	sock_put(sk);
 	return ret;
 }
+
+/* Must be called with rcu read lock held */
+struct mptcp_pm_ops *mptcp_pm_find(enum mptcp_pm_type type)
+{
+	struct mptcp_pm_ops *pm;
+
+	list_for_each_entry_rcu(pm, &mptcp_pm_list, list) {
+		if (pm->type == type)
+			return pm;
+	}
+
+	return NULL;
+}
+
+int mptcp_validate_path_manager(struct mptcp_pm_ops *pm)
+{
+	if (!pm->address_announce && !pm->address_remove &&
+	    !pm->subflow_create && !pm->subflow_destroy &&
+	    !pm->get_local_id && !pm->get_flags && !pm->set_flags) {
+		pr_err("%u does not implement required ops\n", pm->type);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int mptcp_register_path_manager(struct mptcp_pm_ops *pm)
+{
+	int ret;
+
+	ret = mptcp_validate_path_manager(pm);
+	if (ret)
+		return ret;
+
+	spin_lock(&mptcp_pm_list_lock);
+	if (mptcp_pm_find(pm->type)) {
+		spin_unlock(&mptcp_pm_list_lock);
+		return -EEXIST;
+	}
+	list_add_tail_rcu(&pm->list, &mptcp_pm_list);
+	spin_unlock(&mptcp_pm_list_lock);
+
+	pr_debug("userspace_pm type %u registered\n", pm->type);
+	return 0;
+}
+
+void mptcp_unregister_path_manager(struct mptcp_pm_ops *pm)
+{
+	spin_lock(&mptcp_pm_list_lock);
+	list_del_rcu(&pm->list);
+	spin_unlock(&mptcp_pm_list_lock);
+}
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_
 void mptcp_pm_remove_addr_entry(struct mptcp_sock *msk,
 				struct mptcp_pm_addr_entry *entry);
 
+struct mptcp_pm_ops *mptcp_pm_find(enum mptcp_pm_type type);
+int mptcp_validate_path_manager(struct mptcp_pm_ops *pm);
+int mptcp_register_path_manager(struct mptcp_pm_ops *pm);
+void mptcp_unregister_path_manager(struct mptcp_pm_ops *pm);
+
 void mptcp_free_local_addr_list(struct mptcp_sock *msk);
 
 void mptcp_event(enum mptcp_event_type type, const struct mptcp_sock *msk,
-- 
2.43.0

From: Geliang Tang <tanggeliang@kylinos.cn>

This patch defines the original userspace pm as the default path manager,
named mptcp_userspace_pm, and register it in mptcp_pm_data_init().

Extract address_announce() interface of the userspace PM from the handler
of netlink command MPTCP_PM_CMD_ANNOUNCE mptcp_pm_nl_announce_doit(), only
leave the code for obtaining msk through "info" and parsing address entry
in the handler.

Extract address_remove() interface of the userspace PM from the handler
of netlink command MPTCP_PM_CMD_REMOVE mptcp_pm_nl_remove_doit(), only
leave the code for parsing address id and obtaining msk through "info"
in the handler.

Extract subflow_create() interface of the userspace PM from the handler of
netlink command MPTCP_PM_CMD_SUBFLOW_CREATE

mptcp_pm_nl_subflow_create_doit(),

only leave the code for obtaining msk through "info", parsing local address
entry and parsing remote address info in the handler.

Extract subflow_destroy() interface of the userspace PM from the handler of
netlink command MPTCP_PM_CMD_SUBFLOW_DESTROY

mptcp_pm_nl_subflow_destroy_doit(),

only leave the code for obtaining msk through "info", parsing local address
entry and parsing remote address info in the handler.

Extract set_flags() interface of the userspace PM from function
mptcp_userspace_pm_set_flags(), only leave the code for obtaining
msk through "info" in this function.

All fives interfaces are invoked under holding the msk socket lock.

Extract get_local_id() interface of the userspace PM from function
mptcp_userspace_pm_get_local_id(), only leave the code for setting
new address entry in this function.

get_flags() interface of the userspace PM is defined as the same as
the function mptcp_userspace_pm_get_flags(), then this function
becomes a wrapper.

Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
---
 net/mptcp/pm.c           |   1 +
 net/mptcp/pm_userspace.c | 273 ++++++++++++++++++++++++---------------
 net/mptcp/protocol.h     |   1 +
 3 files changed, 172 insertions(+), 103 deletions(-)

diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm.c
+++ b/net/mptcp/pm.c
@@ -XXX,XX +XXX,XX @@ void mptcp_pm_data_init(struct mptcp_sock *msk)
 void __init mptcp_pm_init(void)
 {
 	mptcp_pm_nl_init();
+	mptcp_userspace_pm_init();
 }
diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm_userspace.c
+++ b/net/mptcp/pm_userspace.c
@@ -XXX,XX +XXX,XX @@ mptcp_userspace_pm_lookup_addr_by_id(struct mptcp_sock *msk, unsigned int id)
 	return NULL;
 }
 
-int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
-				    struct mptcp_addr_info *skc)
+static int userspace_pm_get_local_id(struct mptcp_sock *msk,
+				     struct mptcp_pm_addr_entry *skc)
 {
-	struct mptcp_pm_addr_entry *entry = NULL, new_entry;
-	__be16 msk_sport =  ((struct inet_sock *)
-			     inet_sk((struct sock *)msk))->inet_sport;
+	struct mptcp_pm_addr_entry *entry;
 
 	spin_lock_bh(&msk->pm.lock);
-	entry = mptcp_userspace_pm_lookup_addr(msk, skc);
+	entry = mptcp_userspace_pm_lookup_addr(msk, &skc->addr);
 	spin_unlock_bh(&msk->pm.lock);
 	if (entry)
 		return entry->addr.id;
 
+	return mptcp_userspace_pm_append_new_local_addr(msk, skc, true);
+}
+
+int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
+				    struct mptcp_addr_info *skc)
+{
+	__be16 msk_sport = ((struct inet_sock *)
+			    inet_sk((struct sock *)msk))->inet_sport;
+	struct mptcp_pm_addr_entry new_entry;
+
 	memset(&new_entry, 0, sizeof(struct mptcp_pm_addr_entry));
 	new_entry.addr = *skc;
 	new_entry.addr.id = 0;
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
 	if (new_entry.addr.port == msk_sport)
 		new_entry.addr.port = 0;
 
-	return mptcp_userspace_pm_append_new_local_addr(msk, &new_entry, true);
+	return userspace_pm_get_local_id(msk, &new_entry);
 }
 
-u8 mptcp_userspace_pm_get_flags(struct mptcp_sock *msk,
-				struct mptcp_addr_info *skc)
+static u8 userspace_pm_get_flags(struct mptcp_sock *msk,
+				 struct mptcp_addr_info *skc)
 {
 	struct mptcp_pm_addr_entry *entry;
 	u8 flags;
@@ -XXX,XX +XXX,XX @@ u8 mptcp_userspace_pm_get_flags(struct mptcp_sock *msk,
 	return flags;
 }
 
+u8 mptcp_userspace_pm_get_flags(struct mptcp_sock *msk,
+				struct mptcp_addr_info *skc)
+{
+	return userspace_pm_get_flags(msk, skc);
+}
+
 static struct mptcp_sock *mptcp_userspace_pm_get_sock(const struct genl_info *info)
 {
 	struct mptcp_sock *msk;
@@ -XXX,XX +XXX,XX @@ static struct mptcp_sock *mptcp_userspace_pm_get_sock(const struct genl_info *in
 	return msk;
 }
 
+static int userspace_pm_address_announce(struct mptcp_sock *msk,
+					 struct mptcp_pm_addr_entry *local)
+{
+	int err;
+
+	err = mptcp_userspace_pm_append_new_local_addr(msk, local, false);
+	if (err < 0)
+		return err;
+
+	spin_lock_bh(&msk->pm.lock);
+
+	if (mptcp_pm_alloc_anno_list(msk, &local->addr)) {
+		msk->pm.add_addr_signaled++;
+		mptcp_pm_announce_addr(msk, &local->addr, false);
+		mptcp_pm_nl_addr_send_ack(msk);
+	}
+
+	spin_unlock_bh(&msk->pm.lock);
+
+	return 0;
+}
+
 int mptcp_pm_nl_announce_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct mptcp_pm_addr_entry addr_val;
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_announce_doit(struct sk_buff *skb, struct genl_info *info)
 		goto announce_err;
 	}
 
-	err = mptcp_userspace_pm_append_new_local_addr(msk, &addr_val, false);
-	if (err < 0) {
-		NL_SET_ERR_MSG_ATTR(info->extack, addr,
-				    "did not match address and id");
-		goto announce_err;
-	}
-
 	lock_sock(sk);
-	spin_lock_bh(&msk->pm.lock);
-
-	if (mptcp_pm_alloc_anno_list(msk, &addr_val.addr)) {
-		msk->pm.add_addr_signaled++;
-		mptcp_pm_announce_addr(msk, &addr_val.addr, false);
-		mptcp_pm_nl_addr_send_ack(msk);
-	}
-
-	spin_unlock_bh(&msk->pm.lock);
+	err = userspace_pm_address_announce(msk, &addr_val);
 	release_sock(sk);
+	if (err)
+		NL_SET_ERR_MSG_ATTR(info->extack, addr,
+				    "did not match address and id");
 
-	err = 0;
  announce_err:
 	sock_put(sk);
 	return err;
@@ -XXX,XX +XXX,XX @@ void mptcp_pm_remove_addr_entry(struct mptcp_sock *msk,
 	spin_unlock_bh(&msk->pm.lock);
 }
 
+static int userspace_pm_address_remove(struct mptcp_sock *msk, u8 id)
+{
+	struct mptcp_pm_addr_entry *entry;
+
+	if (id == 0)
+		return mptcp_userspace_pm_remove_id_zero_address(msk);
+
+	spin_lock_bh(&msk->pm.lock);
+	entry = mptcp_userspace_pm_lookup_addr_by_id(msk, id);
+	if (!entry) {
+		spin_unlock_bh(&msk->pm.lock);
+		return -EINVAL;
+	}
+
+	list_del_rcu(&entry->list);
+	spin_unlock_bh(&msk->pm.lock);
+
+	mptcp_pm_remove_addr_entry(msk, entry);
+
+	sock_kfree_s((struct sock *)msk, entry, sizeof(*entry));
+
+	return 0;
+}
+
 int mptcp_pm_nl_remove_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct mptcp_pm_addr_entry *match;
 	struct mptcp_sock *msk;
 	struct nlattr *id;
 	int err = -EINVAL;
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_remove_doit(struct sk_buff *skb, struct genl_info *info)
 	sk = (struct sock *)msk;
 
 	lock_sock(sk);
-
-	if (id_val == 0) {
-		err = mptcp_userspace_pm_remove_id_zero_address(msk);
-		release_sock(sk);
-		goto out;
-	}
-
-	spin_lock_bh(&msk->pm.lock);
-	match = mptcp_userspace_pm_lookup_addr_by_id(msk, id_val);
-	if (!match) {
-		spin_unlock_bh(&msk->pm.lock);
-		release_sock(sk);
-		goto out;
-	}
-
-	list_del_rcu(&match->list);
-	spin_unlock_bh(&msk->pm.lock);
-
-	mptcp_pm_remove_addr_entry(msk, match);
-
+	err = userspace_pm_address_remove(msk, id_val);
 	release_sock(sk);
-
-	sock_kfree_s(sk, match, sizeof(*match));
-
-	err = 0;
-out:
 	if (err)
 		NL_SET_ERR_MSG_ATTR_FMT(info->extack, id,
 					"address with id %u not found",
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_remove_doit(struct sk_buff *skb, struct genl_info *info)
 	return err;
 }
 
+static int userspace_pm_subflow_create(struct mptcp_sock *msk,
+				       struct mptcp_pm_addr_entry *entry,
+				       struct mptcp_addr_info *remote)
+{
+	struct sock *sk = (struct sock *)msk;
+	struct mptcp_pm_local local;
+	int err;
+
+	err = mptcp_userspace_pm_append_new_local_addr(msk, entry, false);
+	if (err < 0)
+		return err;
+
+	local.addr = entry->addr;
+	local.flags = entry->flags;
+	local.ifindex = entry->ifindex;
+
+	err = __mptcp_subflow_connect(sk, &local, remote);
+	spin_lock_bh(&msk->pm.lock);
+	if (err)
+		mptcp_userspace_pm_delete_local_addr(msk, entry);
+	else
+		msk->pm.subflows++;
+	spin_unlock_bh(&msk->pm.lock);
+
+	return err;
+}
+
 int mptcp_pm_nl_subflow_create_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct mptcp_pm_addr_entry entry = { 0 };
 	struct mptcp_addr_info addr_r;
 	struct nlattr *raddr, *laddr;
-	struct mptcp_pm_local local;
 	struct mptcp_sock *msk;
 	int err = -EINVAL;
 	struct sock *sk;
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_subflow_create_doit(struct sk_buff *skb, struct genl_info *info)
 		goto create_err;
 	}
 
-	err = mptcp_userspace_pm_append_new_local_addr(msk, &entry, false);
-	if (err < 0) {
-		NL_SET_ERR_MSG_ATTR(info->extack, laddr,
-				    "did not match address and id");
-		goto create_err;
-	}
-
-	local.addr = entry.addr;
-	local.flags = entry.flags;
-	local.ifindex = entry.ifindex;
-
 	lock_sock(sk);
-	err = __mptcp_subflow_connect(sk, &local, &addr_r);
+	err = userspace_pm_subflow_create(msk, &entry, &addr_r);
 	release_sock(sk);
 
 	if (err)
 		GENL_SET_ERR_MSG_FMT(info, "connect error: %d", err);
 
-	spin_lock_bh(&msk->pm.lock);
-	if (err)
-		mptcp_userspace_pm_delete_local_addr(msk, &entry);
-	else
-		msk->pm.subflows++;
-	spin_unlock_bh(&msk->pm.lock);
-
  create_err:
 	sock_put(sk);
 	return err;
@@ -XXX,XX +XXX,XX @@ static struct sock *mptcp_nl_find_ssk(struct mptcp_sock *msk,
 	return NULL;
 }
 
+static int userspace_pm_subflow_destroy(struct mptcp_sock *msk,
+					struct mptcp_pm_addr_entry *local,
+					struct mptcp_addr_info *remote)
+{
+	struct sock *ssk, *sk = (struct sock *)msk;
+
+	ssk = mptcp_nl_find_ssk(msk, &local->addr, remote);
+	if (!ssk)
+		return -ESRCH;
+
+	spin_lock_bh(&msk->pm.lock);
+	mptcp_userspace_pm_delete_local_addr(msk, local);
+	spin_unlock_bh(&msk->pm.lock);
+	mptcp_subflow_shutdown(sk, ssk, RCV_SHUTDOWN | SEND_SHUTDOWN);
+	mptcp_close_ssk(sk, ssk, mptcp_subflow_ctx(ssk));
+	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RMSUBFLOW);
+
+	return 0;
+}
+
 int mptcp_pm_nl_subflow_destroy_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct mptcp_pm_addr_entry addr_l;
 	struct mptcp_addr_info addr_r;
 	struct nlattr *raddr, *laddr;
 	struct mptcp_sock *msk;
-	struct sock *sk, *ssk;
 	int err = -EINVAL;
+	struct sock *sk;
 
 	if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ATTR_ADDR) ||
 	    GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ATTR_ADDR_REMOTE))
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_subflow_destroy_doit(struct sk_buff *skb, struct genl_info *info
 	}
 
 	lock_sock(sk);
-	ssk = mptcp_nl_find_ssk(msk, &addr_l.addr, &addr_r);
-	if (!ssk) {
-		GENL_SET_ERR_MSG(info, "subflow not found");
-		err = -ESRCH;
-		goto release_sock;
-	}
-
-	spin_lock_bh(&msk->pm.lock);
-	mptcp_userspace_pm_delete_local_addr(msk, &addr_l);
-	spin_unlock_bh(&msk->pm.lock);
-	mptcp_subflow_shutdown(sk, ssk, RCV_SHUTDOWN | SEND_SHUTDOWN);
-	mptcp_close_ssk(sk, ssk, mptcp_subflow_ctx(ssk));
-	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RMSUBFLOW);
-release_sock:
+	err = userspace_pm_subflow_destroy(msk, &addr_l, &addr_r);
 	release_sock(sk);
+	if (err)
+		GENL_SET_ERR_MSG(info, "subflow not found");
 
 destroy_err:
 	sock_put(sk);
 	return err;
 }
 
+static int userspace_pm_set_flags(struct mptcp_sock *msk,
+				  struct mptcp_pm_addr_entry *local,
+				  struct mptcp_addr_info *remote)
+{
+	struct mptcp_pm_addr_entry *entry;
+	u8 bkup = 0;
+
+	if (local->flags & MPTCP_PM_ADDR_FLAG_BACKUP)
+		bkup = 1;
+
+	spin_lock_bh(&msk->pm.lock);
+	entry = mptcp_userspace_pm_lookup_addr(msk, &local->addr);
+	if (entry) {
+		if (bkup)
+			entry->flags |= MPTCP_PM_ADDR_FLAG_BACKUP;
+		else
+			entry->flags &= ~MPTCP_PM_ADDR_FLAG_BACKUP;
+	}
+	spin_unlock_bh(&msk->pm.lock);
+
+	return mptcp_pm_nl_mp_prio_send_ack(msk, &local->addr, remote, bkup);
+}
+
 int mptcp_userspace_pm_set_flags(struct mptcp_pm_addr_entry *local,
 				 struct genl_info *info)
 {
 	struct mptcp_addr_info rem = { .family = AF_UNSPEC, };
-	struct mptcp_pm_addr_entry *entry;
 	struct nlattr *attr, *attr_rem;
 	struct mptcp_sock *msk;
 	int ret = -EINVAL;
 	struct sock *sk;
-	u8 bkup = 0;
 
 	if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ATTR_ADDR_REMOTE))
 		return ret;
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_set_flags(struct mptcp_pm_addr_entry *local,
 		goto set_flags_err;
 	}
 
-	if (local->flags & MPTCP_PM_ADDR_FLAG_BACKUP)
-		bkup = 1;
-
-	spin_lock_bh(&msk->pm.lock);
-	entry = mptcp_userspace_pm_lookup_addr(msk, &local->addr);
-	if (entry) {
-		if (bkup)
-			entry->flags |= MPTCP_PM_ADDR_FLAG_BACKUP;
-		else
-			entry->flags &= ~MPTCP_PM_ADDR_FLAG_BACKUP;
-	}
-	spin_unlock_bh(&msk->pm.lock);
-
 	lock_sock(sk);
-	ret = mptcp_pm_nl_mp_prio_send_ack(msk, &local->addr, &rem, bkup);
+	ret = userspace_pm_set_flags(msk, local, &rem);
 	release_sock(sk);
 
 	/* mptcp_pm_nl_mp_prio_send_ack() only fails in one case */
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_get_addr(u8 id, struct mptcp_pm_addr_entry *addr,
 	return ret;
 }
 
+static struct mptcp_pm_ops mptcp_userspace_pm = {
+	.address_announce	= userspace_pm_address_announce,
+	.address_remove		= userspace_pm_address_remove,
+	.subflow_create		= userspace_pm_subflow_create,
+	.subflow_destroy	= userspace_pm_subflow_destroy,
+	.get_local_id		= userspace_pm_get_local_id,
+	.get_flags		= userspace_pm_get_flags,
+	.set_flags		= userspace_pm_set_flags,
+	.type			= MPTCP_PM_TYPE_USERSPACE,
+	.owner			= THIS_MODULE,
+};
+
 /* Must be called with rcu read lock held */
 struct mptcp_pm_ops *mptcp_pm_find(enum mptcp_pm_type type)
 {
@@ -XXX,XX +XXX,XX @@ int mptcp_register_path_manager(struct mptcp_pm_ops *pm)
 
 void mptcp_unregister_path_manager(struct mptcp_pm_ops *pm)
 {
+	if (pm == &mptcp_userspace_pm)
+		return;
+
 	spin_lock(&mptcp_pm_list_lock);
 	list_del_rcu(&pm->list);
 	spin_unlock(&mptcp_pm_list_lock);
 }
+
+void __init mptcp_userspace_pm_init(void)
+{
+	mptcp_register_path_manager(&mptcp_userspace_pm);
+}
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -XXX,XX +XXX,XX @@ static inline u8 subflow_get_local_id(const struct mptcp_subflow_context *subflo
 }
 
 void __init mptcp_pm_nl_init(void);
+void __init mptcp_userspace_pm_init(void);
 void mptcp_pm_nl_work(struct mptcp_sock *msk);
 unsigned int mptcp_pm_get_add_addr_signal_max(const struct mptcp_sock *msk);
 unsigned int mptcp_pm_get_add_addr_accept_max(const struct mptcp_sock *msk);
-- 
2.43.0

From: Geliang Tang <tanggeliang@kylinos.cn>

Add a struct mptcp_pm_ops pointer "ops" in struct mptcp_pm_data, and two
functions mptcp_init_pm() and mptcp_release_pm(), to set and release this
pointer. mptcp_init_pm() is invoked in mptcp_pm_data_reset(), while
mptcp_release_pm() is invoked in __mptcp_destroy_sock().

In this way, different userspace path managers can be initialized through
the pm_type sysctl, and then called into their respective interfaces
through "ops" of "msk->pm".

Signed-off-by: Geliang Tang <tanggeliang@kylinos.cn>
---
 net/mptcp/pm.c           |  4 +++
 net/mptcp/pm_userspace.c | 58 +++++++++++++++++++++++++++++++++++-----
 net/mptcp/protocol.c     |  1 +
 net/mptcp/protocol.h     |  3 +++
 4 files changed, 59 insertions(+), 7 deletions(-)

diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm.c
+++ b/net/mptcp/pm.c
@@ -XXX,XX +XXX,XX @@ void mptcp_pm_data_reset(struct mptcp_sock *msk)
 		WRITE_ONCE(pm->work_pending, 0);
 		WRITE_ONCE(pm->accept_addr, 0);
 		WRITE_ONCE(pm->accept_subflow, 0);
+
+		rcu_read_lock();
+		mptcp_init_pm(msk, mptcp_pm_find(pm_type));
+		rcu_read_unlock();
 	}
 
 	WRITE_ONCE(pm->addr_signal, 0);
diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/pm_userspace.c
+++ b/net/mptcp/pm_userspace.c
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk,
 	if (new_entry.addr.port == msk_sport)
 		new_entry.addr.port = 0;
 
-	return userspace_pm_get_local_id(msk, &new_entry);
+	return msk->pm.ops->get_local_id ?
+	       msk->pm.ops->get_local_id(msk, &new_entry) :
+	       userspace_pm_get_local_id(msk, &new_entry);
 }
 
 static u8 userspace_pm_get_flags(struct mptcp_sock *msk,
@@ -XXX,XX +XXX,XX @@ static u8 userspace_pm_get_flags(struct mptcp_sock *msk,
 u8 mptcp_userspace_pm_get_flags(struct mptcp_sock *msk,
 				struct mptcp_addr_info *skc)
 {
-	return userspace_pm_get_flags(msk, skc);
+	return msk->pm.ops->get_flags ?
+	       msk->pm.ops->get_flags(msk, skc) :
+	       userspace_pm_get_flags(msk, skc);
 }
 
 static struct mptcp_sock *mptcp_userspace_pm_get_sock(const struct genl_info *info)
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_announce_doit(struct sk_buff *skb, struct genl_info *info)
 	}
 
 	lock_sock(sk);
-	err = userspace_pm_address_announce(msk, &addr_val);
+	err = msk->pm.ops->address_announce ?
+	      msk->pm.ops->address_announce(msk, &addr_val) :
+	      userspace_pm_address_announce(msk, &addr_val);
 	release_sock(sk);
 	if (err)
 		NL_SET_ERR_MSG_ATTR(info->extack, addr,
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_remove_doit(struct sk_buff *skb, struct genl_info *info)
 	sk = (struct sock *)msk;
 
 	lock_sock(sk);
-	err = userspace_pm_address_remove(msk, id_val);
+	err = msk->pm.ops->address_remove ?
+	      msk->pm.ops->address_remove(msk, id_val) :
+	      userspace_pm_address_remove(msk, id_val);
 	release_sock(sk);
 	if (err)
 		NL_SET_ERR_MSG_ATTR_FMT(info->extack, id,
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_subflow_create_doit(struct sk_buff *skb, struct genl_info *info)
 	}
 
 	lock_sock(sk);
-	err = userspace_pm_subflow_create(msk, &entry, &addr_r);
+	err = msk->pm.ops->subflow_create ?
+	      msk->pm.ops->subflow_create(msk, &entry, &addr_r) :
+	      userspace_pm_subflow_create(msk, &entry, &addr_r);
 	release_sock(sk);
 
 	if (err)
@@ -XXX,XX +XXX,XX @@ int mptcp_pm_nl_subflow_destroy_doit(struct sk_buff *skb, struct genl_info *info
 	}
 
 	lock_sock(sk);
-	err = userspace_pm_subflow_destroy(msk, &addr_l, &addr_r);
+	err = msk->pm.ops->subflow_destroy ?
+	      msk->pm.ops->subflow_destroy(msk, &addr_l, &addr_r) :
+	      userspace_pm_subflow_destroy(msk, &addr_l, &addr_r);
 	release_sock(sk);
 	if (err)
 		GENL_SET_ERR_MSG(info, "subflow not found");
@@ -XXX,XX +XXX,XX @@ int mptcp_userspace_pm_set_flags(struct mptcp_pm_addr_entry *local,
 	}
 
 	lock_sock(sk);
-	ret = userspace_pm_set_flags(msk, local, &rem);
+	ret = msk->pm.ops->set_flags ?
+	      msk->pm.ops->set_flags(msk, local, &rem) :
+	      userspace_pm_set_flags(msk, local, &rem);
 	release_sock(sk);
 
 	/* mptcp_pm_nl_mp_prio_send_ack() only fails in one case */
@@ -XXX,XX +XXX,XX @@ void mptcp_unregister_path_manager(struct mptcp_pm_ops *pm)
 	spin_unlock(&mptcp_pm_list_lock);
 }
 
+int mptcp_init_pm(struct mptcp_sock *msk, struct mptcp_pm_ops *pm)
+{
+	if (!pm)
+		pm = &mptcp_userspace_pm;
+
+	if (!bpf_try_module_get(pm, pm->owner))
+		return -EBUSY;
+
+	msk->pm.ops = pm;
+	if (msk->pm.ops->init)
+		msk->pm.ops->init(msk);
+
+	pr_debug("userspace_pm type %u initialized\n", msk->pm.ops->type);
+	return 0;
+}
+
+void mptcp_release_pm(struct mptcp_sock *msk)
+{
+	struct mptcp_pm_ops *pm = msk->pm.ops;
+
+	if (!pm)
+		return;
+
+	msk->pm.ops = NULL;
+	if (pm->release)
+		pm->release(msk);
+
+	bpf_module_put(pm, pm->owner);
+}
+
 void __init mptcp_userspace_pm_init(void)
 {
 	mptcp_register_path_manager(&mptcp_userspace_pm);
diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/protocol.c
+++ b/net/mptcp/protocol.c
@@ -XXX,XX +XXX,XX @@ static void __mptcp_destroy_sock(struct sock *sk)
 	sk_stop_timer(sk, &sk->sk_timer);
 	msk->pm.status = 0;
 	mptcp_release_sched(msk);
+	mptcp_release_pm(msk);
 
 	sk->sk_prot->destroy(sk);
 
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index XXXXXXX..XXXXXXX 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -XXX,XX +XXX,XX @@ struct mptcp_pm_data {
 	struct mptcp_addr_info remote;
 	struct list_head anno_list;
 	struct list_head userspace_pm_local_addr_list;
+	struct mptcp_pm_ops *ops;
 
 	spinlock_t	lock;		/*protects the whole PM data */
 
@@ -XXX,XX +XXX,XX @@ struct mptcp_pm_ops *mptcp_pm_find(enum mptcp_pm_type type);
 int mptcp_validate_path_manager(struct mptcp_pm_ops *pm);
 int mptcp_register_path_manager(struct mptcp_pm_ops *pm);
 void mptcp_unregister_path_manager(struct mptcp_pm_ops *pm);
+int mptcp_init_pm(struct mptcp_sock *msk, struct mptcp_pm_ops *pm);
+void mptcp_release_pm(struct mptcp_sock *msk);
 
 void mptcp_free_local_addr_list(struct mptcp_sock *msk);
 
-- 
2.43.0