lkml.org 
[lkml]   [2018]   [Nov]   [27]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patches in this message
/
Date
From
SubjectRe: [PATCH v2 0/4] Static calls

Here's the test with the attached config (A fedora distro with
localmodconfig run against it), with also two patches to implement
tracepoints with static calls. The first makes it where a tracepoint
will call a function pointer to a single callback if there's only one
callback, or an "iterator" which iterates a list of callbacks (when
there are more than one callback associated to a tracepoint).

It adds printks() to where it enables and disables the tracepoints so
expect to see a lot of output when you enable the tracepoints. This is
to verify that it's assigning the right code.

Here's what I did.

1) I first took the config and turned off CONFIG_RETPOLINE and built
v4.20-rc4 with that. I ran this to see what the affect was without
retpolines. I booted that kernel and did the following (which is also
what I did for every kernel):

# trace-cmd start -e all

To get the same affect you could also do:

# echo 1 > /sys/kernel/debug/tracing/events/enable

# perf stat -r 10 /work/c/hackbench 50

The output was this:

No RETPOLINES:

# perf stat -r 10 /work/c/hackbench 50
Time: 1.351
Time: 1.414
Time: 1.319
Time: 1.277
Time: 1.280
Time: 1.305
Time: 1.294
Time: 1.342
Time: 1.319
Time: 1.288

Performance counter stats for '/work/c/hackbench 50' (10 runs):

10,727.44 msec task-clock # 7.397 CPUs utilized ( +- 0.95% )
126,300 context-switches # 11774.138 M/sec ( +- 13.80% )
14,309 cpu-migrations # 1333.973 M/sec ( +- 8.73% )
44,073 page-faults # 4108.652 M/sec ( +- 0.68% )
39,484,799,554 cycles # 3680914.295 GHz ( +- 0.95% )
28,470,896,143 stalled-cycles-frontend # 72.11% frontend cycles idle ( +- 0.95% )
26,521,427,813 instructions # 0.67 insn per cycle
# 1.07 stalled cycles per insn ( +- 0.85% )
4,931,066,096 branches # 459691625.400 M/sec ( +- 0.87% )
19,063,801 branch-misses # 0.39% of all branches ( +- 2.05% )

1.4503 +- 0.0148 seconds time elapsed ( +- 1.02% )

Then I enabled CONFIG_RETPOLINES, built boot and ran it again:

baseline RETPOLINES:

# perf stat -r 10 /work/c/hackbench 50
Time: 1.313
Time: 1.386
Time: 1.335
Time: 1.363
Time: 1.357
Time: 1.369
Time: 1.363
Time: 1.489
Time: 1.357
Time: 1.422

Performance counter stats for '/work/c/hackbench 50' (10 runs):

11,162.24 msec task-clock # 7.383 CPUs utilized ( +- 1.11% )
112,882 context-switches # 10113.153 M/sec ( +- 15.86% )
14,255 cpu-migrations # 1277.103 M/sec ( +- 7.78% )
43,067 page-faults # 3858.393 M/sec ( +- 1.04% )
41,076,270,559 cycles # 3680042.874 GHz ( +- 1.12% )
29,669,137,584 stalled-cycles-frontend # 72.23% frontend cycles idle ( +- 1.21% )
26,647,656,812 instructions # 0.65 insn per cycle
# 1.11 stalled cycles per insn ( +- 0.81% )
5,069,504,923 branches # 454179389.091 M/sec ( +- 0.83% )
99,135,413 branch-misses # 1.96% of all branches ( +- 0.87% )

1.5120 +- 0.0133 seconds time elapsed ( +- 0.88% )


Then I applied the first tracepoint patch to make the change to call
directly (and be able to use static calls later). And tested that.

Added direct calls for trace_events:

# perf stat -r 10 /work/c/hackbench 50
Time: 1.448
Time: 1.386
Time: 1.404
Time: 1.386
Time: 1.344
Time: 1.397
Time: 1.378
Time: 1.351
Time: 1.369
Time: 1.385

Performance counter stats for '/work/c/hackbench 50' (10 runs):

11,249.28 msec task-clock # 7.382 CPUs utilized ( +- 0.64% )
112,058 context-switches # 9961.721 M/sec ( +- 11.15% )
15,535 cpu-migrations # 1381.033 M/sec ( +- 10.34% )
43,673 page-faults # 3882.433 M/sec ( +- 1.14% )
41,407,431,000 cycles # 3681020.455 GHz ( +- 0.63% )
29,842,394,154 stalled-cycles-frontend # 72.07% frontend cycles idle ( +- 0.63% )
26,669,867,181 instructions # 0.64 insn per cycle
# 1.12 stalled cycles per insn ( +- 0.58% )
5,085,122,641 branches # 452055102.392 M/sec ( +- 0.60% )
108,935,006 branch-misses # 2.14% of all branches ( +- 0.57% )

1.5239 +- 0.0139 seconds time elapsed ( +- 0.91% )


Then I added patch 1 and 2, and applied the second attached patch and
ran that:

With static calls:

# perf stat -r 10 /work/c/hackbench 50
Time: 1.407
Time: 1.424
Time: 1.352
Time: 1.355
Time: 1.361
Time: 1.416
Time: 1.453
Time: 1.353
Time: 1.341
Time: 1.439

Performance counter stats for '/work/c/hackbench 50' (10 runs):

11,293.08 msec task-clock # 7.390 CPUs utilized ( +- 0.93% )
125,343 context-switches # 11099.462 M/sec ( +- 11.84% )
15,587 cpu-migrations # 1380.272 M/sec ( +- 8.21% )
43,871 page-faults # 3884.890 M/sec ( +- 1.06% )
41,567,508,330 cycles # 3680918.499 GHz ( +- 0.94% )
29,851,271,023 stalled-cycles-frontend # 71.81% frontend cycles idle ( +- 0.99% )
26,878,085,513 instructions # 0.65 insn per cycle
# 1.11 stalled cycles per insn ( +- 0.72% )
5,125,816,911 branches # 453905346.879 M/sec ( +- 0.74% )
107,643,635 branch-misses # 2.10% of all branches ( +- 0.71% )

1.5282 +- 0.0135 seconds time elapsed ( +- 0.88% )

Then I applied patch 3 and tested that:

With static call trampolines:

# perf stat -r 10 /work/c/hackbench 50
Time: 1.350
Time: 1.333
Time: 1.369
Time: 1.361
Time: 1.375
Time: 1.352
Time: 1.316
Time: 1.336
Time: 1.339
Time: 1.371

Performance counter stats for '/work/c/hackbench 50' (10 runs):

10,964.38 msec task-clock # 7.392 CPUs utilized ( +- 0.41% )
75,986 context-switches # 6930.527 M/sec ( +- 9.23% )
12,464 cpu-migrations # 1136.858 M/sec ( +- 7.93% )
44,476 page-faults # 4056.558 M/sec ( +- 1.12% )
40,354,963,428 cycles # 3680712.468 GHz ( +- 0.42% )
29,057,240,222 stalled-cycles-frontend # 72.00% frontend cycles idle ( +- 0.46% )
26,171,883,339 instructions # 0.65 insn per cycle
# 1.11 stalled cycles per insn ( +- 0.32% )
4,978,193,830 branches # 454053195.523 M/sec ( +- 0.33% )
83,625,127 branch-misses # 1.68% of all branches ( +- 0.33% )

1.48328 +- 0.00515 seconds time elapsed ( +- 0.35% )

And finally I added patch 4 and tested that:

Full static calls:

# perf stat -r 10 /work/c/hackbench 50
Time: 1.302
Time: 1.323
Time: 1.356
Time: 1.325
Time: 1.372
Time: 1.373
Time: 1.319
Time: 1.313
Time: 1.362
Time: 1.322

Performance counter stats for '/work/c/hackbench 50' (10 runs):

10,865.10 msec task-clock # 7.373 CPUs utilized ( +- 0.62% )
88,718 context-switches # 8165.823 M/sec ( +- 10.11% )
13,463 cpu-migrations # 1239.125 M/sec ( +- 8.42% )
44,574 page-faults # 4102.673 M/sec ( +- 0.60% )
39,991,476,585 cycles # 3680897.280 GHz ( +- 0.63% )
28,713,229,777 stalled-cycles-frontend # 71.80% frontend cycles idle ( +- 0.68% )
26,289,703,633 instructions # 0.66 insn per cycle
# 1.09 stalled cycles per insn ( +- 0.44% )
4,983,099,105 branches # 458654631.123 M/sec ( +- 0.45% )
83,719,799 branch-misses # 1.68% of all branches ( +- 0.44% )

1.47364 +- 0.00706 seconds time elapsed ( +- 0.48% )


In summary, we had this:

No RETPOLINES:
1.4503 +- 0.0148 seconds time elapsed ( +- 1.02% )

baseline RETPOLINES:
1.5120 +- 0.0133 seconds time elapsed ( +- 0.88% )

Added direct calls for trace_events:
1.5239 +- 0.0139 seconds time elapsed ( +- 0.91% )

With static calls:
1.5282 +- 0.0135 seconds time elapsed ( +- 0.88% )

With static call trampolines:
1.48328 +- 0.00515 seconds time elapsed ( +- 0.35% )

Full static calls:
1.47364 +- 0.00706 seconds time elapsed ( +- 0.48% )


Adding Retpolines caused a 1.5120 / 1.4503 = 1.0425 ( 4.25% ) slowdown

Trampolines made it into 1.48328 / 1.4503 = 1.0227 ( 2.27% ) slowdown

With full static calls 1.47364 / 1.4503 = 1.0160 ( 1.6% ) slowdown

Going from 4.25 to 1.6 isn't bad, and I think this is very much worth
the effort. I did not expect it to go to 0% as there's a lot of other
places that retpolines cause issues, but this shows that it does help
the tracing code.

I originally did the tests with the development config, which has a
bunch of debugging options enabled (hackbench usually takes over 9
seconds, not the 1.5 that was done here), and the slowdown was closer
to 9% with retpolines. If people want me to do this with that, or I can
send them the config. Or better yet, the code is here, just use your
own configs.

-- Steve
[unhandled content-type:application/octet-stream]From bbbd55dc988fae148ce00cf4512760babbf8a9e0 Mon Sep 17 00:00:00 2001
From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>
Date: Fri, 5 Oct 2018 09:31:43 -0400
Subject: [PATCH] tracepoints: Add a direct call or an iterator

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
---
include/linux/tracepoint-defs.h | 2 +
include/linux/tracepoint.h | 73 ++++++++++++++++++++++-----------
include/trace/define_trace.h | 14 +++----
kernel/tracepoint.c | 29 +++++++++++--
4 files changed, 82 insertions(+), 36 deletions(-)

diff --git a/include/linux/tracepoint-defs.h b/include/linux/tracepoint-defs.h
index 49ba9cde7e4b..8013fc442964 100644
--- a/include/linux/tracepoint-defs.h
+++ b/include/linux/tracepoint-defs.h
@@ -30,6 +30,8 @@ struct tracepoint_func {
struct tracepoint {
const char *name; /* Tracepoint name */
struct static_key key;
+ void **function_ptr;
+ void *iterator;
int (*regfunc)(void);
void (*unregfunc)(void);
struct tracepoint_func __rcu *funcs;
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 538ba1a58f5b..b60d547c44ba 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -94,7 +94,9 @@ extern int syscall_regfunc(void);
extern void syscall_unregfunc(void);
#endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */

+#ifndef PARAMS
#define PARAMS(args...) args
+#endif

#define TRACE_DEFINE_ENUM(x)
#define TRACE_DEFINE_SIZEOF(x)
@@ -161,12 +163,11 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
* as "(void *, void)". The DECLARE_TRACE_NOARGS() will pass in just
* "void *data", where as the DECLARE_TRACE() will pass in "void *data, proto".
*/
-#define __DO_TRACE(tp, proto, args, cond, rcuidle) \
+#define __DO_TRACE(name, proto, args, cond, rcuidle) \
do { \
struct tracepoint_func *it_func_ptr; \
- void *it_func; \
- void *__data; \
int __maybe_unused idx = 0; \
+ void *__data; \
\
if (!(cond)) \
return; \
@@ -186,14 +187,12 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
rcu_irq_enter_irqson(); \
} \
\
- it_func_ptr = rcu_dereference_raw((tp)->funcs); \
- \
+ it_func_ptr = \
+ rcu_dereference_raw((&__tracepoint_##name)->funcs); \
if (it_func_ptr) { \
- do { \
- it_func = (it_func_ptr)->func; \
- __data = (it_func_ptr)->data; \
- ((void(*)(proto))(it_func))(args); \
- } while ((++it_func_ptr)->func); \
+ __data = (it_func_ptr)->data; \
+ /* printk("%pS\n", __tp_func_##name); */ \
+ __tp_func_##name(args); \
} \
\
if (rcuidle) { \
@@ -209,7 +208,7 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
static inline void trace_##name##_rcuidle(proto) \
{ \
if (static_key_false(&__tracepoint_##name.key)) \
- __DO_TRACE(&__tracepoint_##name, \
+ __DO_TRACE(name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
TP_CONDITION(cond), 1); \
@@ -231,11 +230,13 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
* poking RCU a bit.
*/
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
+ extern void __tracepoint_iter_##name(data_proto); \
+ extern void (*__tp_func_##name)(data_proto); \
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
if (static_key_false(&__tracepoint_##name.key)) \
- __DO_TRACE(&__tracepoint_##name, \
+ __DO_TRACE(name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
TP_CONDITION(cond), 0); \
@@ -281,21 +282,43 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
* structures, so we create an array of pointers that will be used for iteration
* on the tracepoints.
*/
-#define DEFINE_TRACE_FN(name, reg, unreg) \
- static const char __tpstrtab_##name[] \
- __attribute__((section("__tracepoints_strings"))) = #name; \
- struct tracepoint __tracepoint_##name \
- __attribute__((section("__tracepoints"), used)) = \
- { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
- __TRACEPOINT_ENTRY(name);
+#define DEFINE_TRACE_FN(name, reg, unreg, proto, args) \
+ static const char __tpstrtab_##name[] \
+ __attribute__((section("__tracepoints_strings"))) = #name; \
+ void __tracepoint_iter_##name(void *__data, proto); \
+ void (*__tp_func_##name)(void *__data, proto) = \
+ __tracepoint_iter_##name; \
+ struct tracepoint __tracepoint_##name \
+ __attribute__((section("__tracepoints"), used)) = \
+ { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, \
+ (void **)&__tp_func_##name, __tracepoint_iter_##name, \
+ reg, unreg, NULL }; \
+ __TRACEPOINT_ENTRY(name); \
+ void __tracepoint_iter_##name(void *__data, proto) \
+ { \
+ struct tracepoint_func *it_func_ptr; \
+ void *it_func; \
+ \
+ it_func_ptr = \
+ rcu_dereference_raw((&__tracepoint_##name)->funcs); \
+ do { \
+ it_func = (it_func_ptr)->func; \
+ __data = (it_func_ptr)->data; \
+ ((void(*)(void *, proto))(it_func))(__data, args); \
+ } while ((++it_func_ptr)->func); \
+ return 0; \
+ }

-#define DEFINE_TRACE(name) \
- DEFINE_TRACE_FN(name, NULL, NULL);
+#define DEFINE_TRACE(name, proto, args) \
+ DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));

#define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \
- EXPORT_SYMBOL_GPL(__tracepoint_##name)
+ EXPORT_SYMBOL_GPL(__tracepoint_##name); \
+ EXPORT_SYMBOL_GPL(__tp_func_##name)
#define EXPORT_TRACEPOINT_SYMBOL(name) \
- EXPORT_SYMBOL(__tracepoint_##name)
+ EXPORT_SYMBOL(__tracepoint_##name); \
+ EXPORT_SYMBOL(__tp_func_##name)
+

#else /* !TRACEPOINTS_ENABLED */
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
@@ -324,8 +347,8 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
return false; \
}

-#define DEFINE_TRACE_FN(name, reg, unreg)
-#define DEFINE_TRACE(name)
+#define DEFINE_TRACE_FN(name, reg, unreg, proto, args)
+#define DEFINE_TRACE(name, proto, args)
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name)
#define EXPORT_TRACEPOINT_SYMBOL(name)

diff --git a/include/trace/define_trace.h b/include/trace/define_trace.h
index cb30c5532144..c19aea44efb2 100644
--- a/include/trace/define_trace.h
+++ b/include/trace/define_trace.h
@@ -25,7 +25,7 @@

#undef TRACE_EVENT
#define TRACE_EVENT(name, proto, args, tstruct, assign, print) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))

#undef TRACE_EVENT_CONDITION
#define TRACE_EVENT_CONDITION(name, proto, args, cond, tstruct, assign, print) \
@@ -39,24 +39,24 @@
#undef TRACE_EVENT_FN
#define TRACE_EVENT_FN(name, proto, args, tstruct, \
assign, print, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))

#undef TRACE_EVENT_FN_COND
#define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \
assign, print, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))

#undef DEFINE_EVENT
#define DEFINE_EVENT(template, name, proto, args) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))

#undef DEFINE_EVENT_FN
#define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))

#undef DEFINE_EVENT_PRINT
#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))

#undef DEFINE_EVENT_CONDITION
#define DEFINE_EVENT_CONDITION(template, name, proto, args, cond) \
@@ -64,7 +64,7 @@

#undef DECLARE_TRACE
#define DECLARE_TRACE(name, proto, args) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))

#undef TRACE_INCLUDE
#undef __TRACE_INCLUDE
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index a3be42304485..7e1f34edd1d7 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -140,7 +140,7 @@ static void debug_print_probes(struct tracepoint_func *funcs)

static struct tracepoint_func *
func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func,
- int prio)
+ int prio, int *tot_probes)
{
struct tracepoint_func *old, *new;
int nr_probes = 0;
@@ -183,11 +183,12 @@ func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func,
new[nr_probes + 1].func = NULL;
*funcs = new;
debug_print_probes(*funcs);
+ *tot_probes = nr_probes + 1;
return old;
}

static void *func_remove(struct tracepoint_func **funcs,
- struct tracepoint_func *tp_func)
+ struct tracepoint_func *tp_func, int *left)
{
int nr_probes = 0, nr_del = 0, i;
struct tracepoint_func *old, *new;
@@ -241,6 +242,7 @@ static int tracepoint_add_func(struct tracepoint *tp,
struct tracepoint_func *func, int prio)
{
struct tracepoint_func *old, *tp_funcs;
+ int probes = 0;
int ret;

if (tp->regfunc && !static_key_enabled(&tp->key)) {
@@ -251,7 +253,7 @@ static int tracepoint_add_func(struct tracepoint *tp,

tp_funcs = rcu_dereference_protected(tp->funcs,
lockdep_is_held(&tracepoints_mutex));
- old = func_add(&tp_funcs, func, prio);
+ old = func_add(&tp_funcs, func, prio, &probes);
if (IS_ERR(old)) {
WARN_ON_ONCE(PTR_ERR(old) != -ENOMEM);
return PTR_ERR(old);
@@ -266,6 +268,15 @@ static int tracepoint_add_func(struct tracepoint *tp,
rcu_assign_pointer(tp->funcs, tp_funcs);
if (!static_key_enabled(&tp->key))
static_key_slow_inc(&tp->key);
+
+ if (probes == 1) {
+ printk("make direct call to %pS\n", tp_funcs->func);
+ *tp->function_ptr = tp_funcs->func;
+ } else {
+ printk("[%d] make call to iterator %pS\n", probes, tp->iterator);
+ *tp->function_ptr = tp->iterator;
+ }
+
release_probes(old);
return 0;
}
@@ -280,10 +291,11 @@ static int tracepoint_remove_func(struct tracepoint *tp,
struct tracepoint_func *func)
{
struct tracepoint_func *old, *tp_funcs;
+ int probes_left = 0;

tp_funcs = rcu_dereference_protected(tp->funcs,
lockdep_is_held(&tracepoints_mutex));
- old = func_remove(&tp_funcs, func);
+ old = func_remove(&tp_funcs, func, &probes_left);
if (IS_ERR(old)) {
WARN_ON_ONCE(PTR_ERR(old) != -ENOMEM);
return PTR_ERR(old);
@@ -297,6 +309,15 @@ static int tracepoint_remove_func(struct tracepoint *tp,
if (static_key_enabled(&tp->key))
static_key_slow_dec(&tp->key);
}
+
+ if (probes_left == 1) {
+ printk("make direct call to %pS\n", tp_funcs->func);
+ *tp->function_ptr = tp_funcs->func;
+ } else {
+ printk("[%d] make call to iterator %pS\n", probes_left, tp->iterator);
+ *tp->function_ptr = tp->iterator;
+ }
+
rcu_assign_pointer(tp->funcs, tp_funcs);
release_probes(old);
return 0;
--
2.19.1
From 341d8326ef359ebfdf7923e01e225f653111c304 Mon Sep 17 00:00:00 2001
From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>
Date: Fri, 5 Oct 2018 09:31:43 -0400
Subject: [PATCH] tracepoints: Implement it with dynamic functions

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
---
include/linux/tracepoint-defs.h | 4 +++-
include/linux/tracepoint.h | 24 ++++++++++++------------
kernel/tracepoint.c | 8 ++++----
3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/include/linux/tracepoint-defs.h b/include/linux/tracepoint-defs.h
index 8013fc442964..ae16672bea61 100644
--- a/include/linux/tracepoint-defs.h
+++ b/include/linux/tracepoint-defs.h
@@ -11,6 +11,8 @@
#include <linux/atomic.h>
#include <linux/static_key.h>

+struct static_call_key;
+
struct trace_print_flags {
unsigned long mask;
const char *name;
@@ -30,7 +32,7 @@ struct tracepoint_func {
struct tracepoint {
const char *name; /* Tracepoint name */
struct static_key key;
- void **function_ptr;
+ struct static_call_key *static_call_key;
void *iterator;
int (*regfunc)(void);
void (*unregfunc)(void);
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index b60d547c44ba..e0d787a7e68b 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -21,6 +21,7 @@
#include <linux/cpumask.h>
#include <linux/rcupdate.h>
#include <linux/tracepoint-defs.h>
+#include <linux/static_call.h>

struct module;
struct tracepoint;
@@ -191,8 +192,7 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
rcu_dereference_raw((&__tracepoint_##name)->funcs); \
if (it_func_ptr) { \
__data = (it_func_ptr)->data; \
- /* printk("%pS\n", __tp_func_##name); */ \
- __tp_func_##name(args); \
+ static_call(__tp_func_##name, args); \
} \
\
if (rcuidle) { \
@@ -230,8 +230,8 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
* poking RCU a bit.
*/
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
- extern void __tracepoint_iter_##name(data_proto); \
- extern void (*__tp_func_##name)(data_proto); \
+ extern int __tracepoint_iter_##name(data_proto); \
+ DECLARE_STATIC_CALL(__tp_func_##name, __tracepoint_iter_##name); \
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
@@ -285,16 +285,15 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
#define DEFINE_TRACE_FN(name, reg, unreg, proto, args) \
static const char __tpstrtab_##name[] \
__attribute__((section("__tracepoints_strings"))) = #name; \
- void __tracepoint_iter_##name(void *__data, proto); \
- void (*__tp_func_##name)(void *__data, proto) = \
- __tracepoint_iter_##name; \
+ int __tracepoint_iter_##name(void *__data, proto); \
+ extern struct static_call_key __tp_func_##name; \
struct tracepoint __tracepoint_##name \
__attribute__((section("__tracepoints"), used)) = \
{ __tpstrtab_##name, STATIC_KEY_INIT_FALSE, \
- (void **)&__tp_func_##name, __tracepoint_iter_##name, \
+ &__tp_func_##name, __tracepoint_iter_##name, \
reg, unreg, NULL }; \
__TRACEPOINT_ENTRY(name); \
- void __tracepoint_iter_##name(void *__data, proto) \
+ int __tracepoint_iter_##name(void *__data, proto) \
{ \
struct tracepoint_func *it_func_ptr; \
void *it_func; \
@@ -307,17 +306,18 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
((void(*)(void *, proto))(it_func))(__data, args); \
} while ((++it_func_ptr)->func); \
return 0; \
- }
+ } \
+ DEFINE_STATIC_CALL(__tp_func_##name, __tracepoint_iter_##name);

#define DEFINE_TRACE(name, proto, args) \
DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));

#define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \
EXPORT_SYMBOL_GPL(__tracepoint_##name); \
- EXPORT_SYMBOL_GPL(__tp_func_##name)
+ EXPORT_STATIC_CALL_GPL(__tp_func_##name)
#define EXPORT_TRACEPOINT_SYMBOL(name) \
EXPORT_SYMBOL(__tracepoint_##name); \
- EXPORT_SYMBOL(__tp_func_##name)
+ EXPORT_STATIC_CALL(__tp_func_##name)


#else /* !TRACEPOINTS_ENABLED */
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index 7e1f34edd1d7..4105bdd872f9 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -271,10 +271,10 @@ static int tracepoint_add_func(struct tracepoint *tp,

if (probes == 1) {
printk("make direct call to %pS\n", tp_funcs->func);
- *tp->function_ptr = tp_funcs->func;
+ __static_call_update(tp->static_call_key, tp_funcs->func);
} else {
printk("[%d] make call to iterator %pS\n", probes, tp->iterator);
- *tp->function_ptr = tp->iterator;
+ __static_call_update(tp->static_call_key, tp->iterator);
}

release_probes(old);
@@ -312,10 +312,10 @@ static int tracepoint_remove_func(struct tracepoint *tp,

if (probes_left == 1) {
printk("make direct call to %pS\n", tp_funcs->func);
- *tp->function_ptr = tp_funcs->func;
+ __static_call_update(tp->static_call_key, tp_funcs->func);
} else {
printk("[%d] make call to iterator %pS\n", probes_left, tp->iterator);
- *tp->function_ptr = tp->iterator;
+ __static_call_update(tp->static_call_key, tp->iterator);
}

rcu_assign_pointer(tp->funcs, tp_funcs);
--
2.19.1
\
 
 \ /
  Last update: 2018-11-26 21:55    [W:0.174 / U:8.120 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site