0856a30409
Patch series 109f6e39..7361c36c back in 2.6.36 added functionality to allow credentials to work across pid namespaces for packets sent via UNIX sockets. However, the atomic reference counts on pid and credentials caused plenty of cache bouncing when there are numerous threads of the same pid sharing a UNIX socket. This patch mitigates the problem by eliminating extraneous reference counts on pid and credentials on both send and receive path of UNIX sockets. I found a 2x improvement in hackbench's threaded case. On the receive path in unix_dgram_recvmsg, currently there is an increment of reference count on pid and credentials in scm_set_cred. Then there are two decrement of the reference counts. Once in scm_recv and once when skb_free_datagram call skb->destructor function unix_destruct_scm. One pair of increment and decrement of ref count on pid and credentials can be eliminated from the receive path. Until we destroy the skb, we already set a reference when we created the skb on the send side. On the send path, there are two increments of ref count on pid and credentials, once in scm_send and once in unix_scm_to_skb. Then there is a decrement of the reference counts in scm_destroy's call to scm_destroy_cred at the end of unix_dgram_sendmsg functions. One pair of increment and decrement of the reference counts can be removed so we only need to increment the ref counts once. By incorporating these changes, for hackbench running on a 4 socket NHM-EX machine with 40 cores, the execution of hackbench on 50 groups of 20 threads sped up by factor of 2. Hackbench command used for testing: ./hackbench 50 thread 2000 Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com> Signed-off-by: David S. Miller <davem@davemloft.net>
146 lines
3.5 KiB
C
146 lines
3.5 KiB
C
#ifndef __LINUX_NET_SCM_H
|
|
#define __LINUX_NET_SCM_H
|
|
|
|
#include <linux/limits.h>
|
|
#include <linux/net.h>
|
|
#include <linux/security.h>
|
|
#include <linux/pid.h>
|
|
#include <linux/nsproxy.h>
|
|
|
|
/* Well, we should have at least one descriptor open
|
|
* to accept passed FDs 8)
|
|
*/
|
|
#define SCM_MAX_FD 253
|
|
|
|
struct scm_fp_list {
|
|
struct list_head list;
|
|
short count;
|
|
short max;
|
|
struct file *fp[SCM_MAX_FD];
|
|
};
|
|
|
|
struct scm_cookie {
|
|
struct pid *pid; /* Skb credentials */
|
|
const struct cred *cred;
|
|
struct scm_fp_list *fp; /* Passed files */
|
|
struct ucred creds; /* Skb credentials */
|
|
#ifdef CONFIG_SECURITY_NETWORK
|
|
u32 secid; /* Passed security ID */
|
|
#endif
|
|
};
|
|
|
|
extern void scm_detach_fds(struct msghdr *msg, struct scm_cookie *scm);
|
|
extern void scm_detach_fds_compat(struct msghdr *msg, struct scm_cookie *scm);
|
|
extern int __scm_send(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm);
|
|
extern void __scm_destroy(struct scm_cookie *scm);
|
|
extern struct scm_fp_list * scm_fp_dup(struct scm_fp_list *fpl);
|
|
|
|
#ifdef CONFIG_SECURITY_NETWORK
|
|
static __inline__ void unix_get_peersec_dgram(struct socket *sock, struct scm_cookie *scm)
|
|
{
|
|
security_socket_getpeersec_dgram(sock, NULL, &scm->secid);
|
|
}
|
|
#else
|
|
static __inline__ void unix_get_peersec_dgram(struct socket *sock, struct scm_cookie *scm)
|
|
{ }
|
|
#endif /* CONFIG_SECURITY_NETWORK */
|
|
|
|
static __inline__ void scm_set_cred(struct scm_cookie *scm,
|
|
struct pid *pid, const struct cred *cred)
|
|
{
|
|
scm->pid = get_pid(pid);
|
|
scm->cred = get_cred(cred);
|
|
cred_to_ucred(pid, cred, &scm->creds);
|
|
}
|
|
|
|
static __inline__ void scm_set_cred_noref(struct scm_cookie *scm,
|
|
struct pid *pid, const struct cred *cred)
|
|
{
|
|
scm->pid = pid;
|
|
scm->cred = cred;
|
|
cred_to_ucred(pid, cred, &scm->creds);
|
|
}
|
|
|
|
static __inline__ void scm_destroy_cred(struct scm_cookie *scm)
|
|
{
|
|
put_pid(scm->pid);
|
|
scm->pid = NULL;
|
|
|
|
if (scm->cred)
|
|
put_cred(scm->cred);
|
|
scm->cred = NULL;
|
|
}
|
|
|
|
static __inline__ void scm_destroy(struct scm_cookie *scm)
|
|
{
|
|
scm_destroy_cred(scm);
|
|
if (scm && scm->fp)
|
|
__scm_destroy(scm);
|
|
}
|
|
|
|
static __inline__ void scm_release(struct scm_cookie *scm)
|
|
{
|
|
/* keep ref on pid and cred */
|
|
scm->pid = NULL;
|
|
scm->cred = NULL;
|
|
if (scm->fp)
|
|
__scm_destroy(scm);
|
|
}
|
|
|
|
static __inline__ int scm_send(struct socket *sock, struct msghdr *msg,
|
|
struct scm_cookie *scm)
|
|
{
|
|
scm_set_cred(scm, task_tgid(current), current_cred());
|
|
scm->fp = NULL;
|
|
unix_get_peersec_dgram(sock, scm);
|
|
if (msg->msg_controllen <= 0)
|
|
return 0;
|
|
return __scm_send(sock, msg, scm);
|
|
}
|
|
|
|
#ifdef CONFIG_SECURITY_NETWORK
|
|
static inline void scm_passec(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm)
|
|
{
|
|
char *secdata;
|
|
u32 seclen;
|
|
int err;
|
|
|
|
if (test_bit(SOCK_PASSSEC, &sock->flags)) {
|
|
err = security_secid_to_secctx(scm->secid, &secdata, &seclen);
|
|
|
|
if (!err) {
|
|
put_cmsg(msg, SOL_SOCKET, SCM_SECURITY, seclen, secdata);
|
|
security_release_secctx(secdata, seclen);
|
|
}
|
|
}
|
|
}
|
|
#else
|
|
static inline void scm_passec(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm)
|
|
{ }
|
|
#endif /* CONFIG_SECURITY_NETWORK */
|
|
|
|
static __inline__ void scm_recv(struct socket *sock, struct msghdr *msg,
|
|
struct scm_cookie *scm, int flags)
|
|
{
|
|
if (!msg->msg_control) {
|
|
if (test_bit(SOCK_PASSCRED, &sock->flags) || scm->fp)
|
|
msg->msg_flags |= MSG_CTRUNC;
|
|
if (scm && scm->fp)
|
|
__scm_destroy(scm);
|
|
return;
|
|
}
|
|
|
|
if (test_bit(SOCK_PASSCRED, &sock->flags))
|
|
put_cmsg(msg, SOL_SOCKET, SCM_CREDENTIALS, sizeof(scm->creds), &scm->creds);
|
|
|
|
scm_passec(sock, msg, scm);
|
|
|
|
if (!scm->fp)
|
|
return;
|
|
|
|
scm_detach_fds(msg, scm);
|
|
}
|
|
|
|
|
|
#endif /* __LINUX_NET_SCM_H */
|
|
|