GitHub-Mirror/linux-bl808
2
0
Fork 0
mirror of https://github.com/Fishwaldo/linux-bl808.git synced 2025-04-09 07:43:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

udp: avoid a cache miss on dequeue

Browse source 
Since UDP no more uses sk->destructor, we can clear completely
the skb head state before enqueuing. Amend and use
skb_release_head_state() for that.

All head states share a single cacheline, which is not
normally used/accesses on dequeue. We can avoid entirely accessing
such cacheline implementing and using in the UDP code a specialized
skb free helper which ignores the skb head state.

This saves a cacheline miss at skb deallocation time.

v1 -> v2:
  replaced secpath_reset() with skb_release_head_state()

Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Paolo Abeni 2017-06-12 11:23:42 +02:00 • committed by David S. Miller
parent 3889a803e1
commit 0a463c78d2
3 changed files with 27 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
include/linux/skbuff.h
View file

@ -880,10 +880,12 @@ static inline bool skb_unref(struct sk_buff *skb)
return true; return true;
} }
void skb_release_head_state(struct sk_buff *skb);
void kfree_skb(struct sk_buff *skb); void kfree_skb(struct sk_buff *skb);
void kfree_skb_list(struct sk_buff *segs); void kfree_skb_list(struct sk_buff *segs);
void skb_tx_error(struct sk_buff *skb); void skb_tx_error(struct sk_buff *skb);
void consume_skb(struct sk_buff *skb); void consume_skb(struct sk_buff *skb);
void consume_stateless_skb(struct sk_buff *skb);
void __kfree_skb(struct sk_buff *skb); void __kfree_skb(struct sk_buff *skb);
extern struct kmem_cache *skbuff_head_cache; extern struct kmem_cache *skbuff_head_cache;

24
net/core/skbuff.c
View file

@ -643,12 +643,10 @@ fastpath:
kmem_cache_free(skbuff_fclone_cache, fclones); kmem_cache_free(skbuff_fclone_cache, fclones);
} }
static void skb_release_head_state(struct sk_buff *skb) void skb_release_head_state(struct sk_buff *skb)
{ {
skb_dst_drop(skb); skb_dst_drop(skb);
#ifdef CONFIG_XFRM secpath_reset(skb);
secpath_put(skb->sp);
#endif
if (skb->destructor) { if (skb->destructor) {
WARN_ON(in_irq()); WARN_ON(in_irq());
skb->destructor(skb); skb->destructor(skb);
@ -751,6 +749,24 @@ void consume_skb(struct sk_buff *skb)
} }
EXPORT_SYMBOL(consume_skb); EXPORT_SYMBOL(consume_skb);
/**
* consume_stateless_skb - free an skbuff, assuming it is stateless
* @skb: buffer to free
*
* Works like consume_skb(), but this variant assumes that all the head
* states have been already dropped.
*/
void consume_stateless_skb(struct sk_buff *skb)
{
if (!skb_unref(skb))
return;
trace_consume_skb(skb);
if (likely(skb->head))
skb_release_data(skb);
kfree_skbmem(skb);
}
void __kfree_skb_flush(void) void __kfree_skb_flush(void)
{ {
struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);

6
net/ipv4/udp.c
View file

@ -1359,7 +1359,8 @@ void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
sk_peek_offset_bwd(sk, len); sk_peek_offset_bwd(sk, len);
unlock_sock_fast(sk, slow); unlock_sock_fast(sk, slow);
} }
consume_skb(skb);
consume_stateless_skb(skb);
} }
EXPORT_SYMBOL_GPL(skb_consume_udp); EXPORT_SYMBOL_GPL(skb_consume_udp);
@ -1739,6 +1740,9 @@ static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
sk_mark_napi_id_once(sk, skb); sk_mark_napi_id_once(sk, skb);
} }
/* clear all pending head states while they are hot in the cache */
skb_release_head_state(skb);
rc = __udp_enqueue_schedule_skb(sk, skb); rc = __udp_enqueue_schedule_skb(sk, skb);
if (rc < 0) { if (rc < 0) {
int is_udplite = IS_UDPLITE(sk); int is_udplite = IS_UDPLITE(sk);