X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=net%2Fipv4%2Finet_hashtables.c;h=8cd1ad9b91118248e2fb8243afc2c7f3e413c6ed;hb=fc8717baa8f52dd8d1b90df9008300ef3ec794ed;hp=67704da04fc4ca98d88eeaae7db3e53d1a6157aa;hpb=cb834e7cc0e8b223386f219c1d1986846c87c55f;p=linux-2.6 diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index 67704da04f..8cd1ad9b91 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -28,12 +28,14 @@ * The bindhash mutex for snum's hash chain must be held here. */ struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep, + struct net *net, struct inet_bind_hashbucket *head, const unsigned short snum) { struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC); if (tb != NULL) { + tb->ib_net = net; tb->port = snum; tb->fastreuse = 0; INIT_HLIST_HEAD(&tb->owners); @@ -64,8 +66,9 @@ void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, /* * Get rid of any references to a local port held by the given sock. */ -static void __inet_put_port(struct inet_hashinfo *hashinfo, struct sock *sk) +static void __inet_put_port(struct sock *sk) { + struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; const int bhash = inet_bhashfn(inet_sk(sk)->num, hashinfo->bhash_size); struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash]; struct inet_bind_bucket *tb; @@ -79,10 +82,10 @@ static void __inet_put_port(struct inet_hashinfo *hashinfo, struct sock *sk) spin_unlock(&head->lock); } -void inet_put_port(struct inet_hashinfo *hashinfo, struct sock *sk) +void inet_put_port(struct sock *sk) { local_bh_disable(); - __inet_put_port(hashinfo, sk); + __inet_put_port(sk); local_bh_enable(); } @@ -96,6 +99,7 @@ EXPORT_SYMBOL(inet_put_port); * exclusive lock release). It should be ifdefed really. */ void inet_listen_wlock(struct inet_hashinfo *hashinfo) + __acquires(hashinfo->lhash_lock) { write_lock(&hashinfo->lhash_lock); @@ -116,15 +120,14 @@ void inet_listen_wlock(struct inet_hashinfo *hashinfo) } } -EXPORT_SYMBOL(inet_listen_wlock); - /* * Don't inline this cruft. Here are some nice properties to exploit here. The * BSD API does not allow a listening sock to specify the remote port nor the * remote address for the connection. So always assume those are both * wildcarded during the search since they can never be otherwise. */ -static struct sock *inet_lookup_listener_slow(const struct hlist_head *head, +static struct sock *inet_lookup_listener_slow(struct net *net, + const struct hlist_head *head, const __be32 daddr, const unsigned short hnum, const int dif) @@ -136,7 +139,8 @@ static struct sock *inet_lookup_listener_slow(const struct hlist_head *head, sk_for_each(sk, node, head) { const struct inet_sock *inet = inet_sk(sk); - if (inet->num == hnum && !ipv6_only_sock(sk)) { + if (sk->sk_net == net && inet->num == hnum && + !ipv6_only_sock(sk)) { const __be32 rcv_saddr = inet->rcv_saddr; int score = sk->sk_family == PF_INET ? 1 : 0; @@ -162,7 +166,8 @@ static struct sock *inet_lookup_listener_slow(const struct hlist_head *head, } /* Optimize the common listener case. */ -struct sock *__inet_lookup_listener(struct inet_hashinfo *hashinfo, +struct sock *__inet_lookup_listener(struct net *net, + struct inet_hashinfo *hashinfo, const __be32 daddr, const unsigned short hnum, const int dif) { @@ -177,9 +182,9 @@ struct sock *__inet_lookup_listener(struct inet_hashinfo *hashinfo, if (inet->num == hnum && !sk->sk_node.next && (!inet->rcv_saddr || inet->rcv_saddr == daddr) && (sk->sk_family == PF_INET || !ipv6_only_sock(sk)) && - !sk->sk_bound_dev_if) + !sk->sk_bound_dev_if && sk->sk_net == net) goto sherry_cache; - sk = inet_lookup_listener_slow(head, daddr, hnum, dif); + sk = inet_lookup_listener_slow(net, head, daddr, hnum, dif); } if (sk) { sherry_cache: @@ -190,6 +195,47 @@ sherry_cache: } EXPORT_SYMBOL_GPL(__inet_lookup_listener); +struct sock * __inet_lookup_established(struct net *net, + struct inet_hashinfo *hashinfo, + const __be32 saddr, const __be16 sport, + const __be32 daddr, const u16 hnum, + const int dif) +{ + INET_ADDR_COOKIE(acookie, saddr, daddr) + const __portpair ports = INET_COMBINED_PORTS(sport, hnum); + struct sock *sk; + const struct hlist_node *node; + /* Optimize here for direct hit, only listening connections can + * have wildcards anyways. + */ + unsigned int hash = inet_ehashfn(daddr, hnum, saddr, sport); + struct inet_ehash_bucket *head = inet_ehash_bucket(hashinfo, hash); + rwlock_t *lock = inet_ehash_lockp(hashinfo, hash); + + prefetch(head->chain.first); + read_lock(lock); + sk_for_each(sk, node, &head->chain) { + if (INET_MATCH(sk, net, hash, acookie, + saddr, daddr, ports, dif)) + goto hit; /* You sunk my battleship! */ + } + + /* Must check for a TIME_WAIT'er before going to listener hash. */ + sk_for_each(sk, node, &head->twchain) { + if (INET_TW_MATCH(sk, net, hash, acookie, + saddr, daddr, ports, dif)) + goto hit; + } + sk = NULL; +out: + read_unlock(lock); + return sk; +hit: + sock_hold(sk); + goto out; +} +EXPORT_SYMBOL_GPL(__inet_lookup_established); + /* called with local bh disabled */ static int __inet_check_established(struct inet_timewait_death_row *death_row, struct sock *sk, __u16 lport, @@ -208,6 +254,7 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row, struct sock *sk2; const struct hlist_node *node; struct inet_timewait_sock *tw; + struct net *net = sk->sk_net; prefetch(head->chain.first); write_lock(lock); @@ -216,7 +263,8 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row, sk_for_each(sk2, node, &head->twchain) { tw = inet_twsk(sk2); - if (INET_TW_MATCH(sk2, hash, acookie, saddr, daddr, ports, dif)) { + if (INET_TW_MATCH(sk2, net, hash, acookie, + saddr, daddr, ports, dif)) { if (twsk_unique(sk, sk2, twp)) goto unique; else @@ -227,7 +275,8 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row, /* And established part... */ sk_for_each(sk2, node, &head->chain) { - if (INET_MATCH(sk2, hash, acookie, saddr, daddr, ports, dif)) + if (INET_MATCH(sk2, net, hash, acookie, + saddr, daddr, ports, dif)) goto not_unique; } @@ -239,7 +288,7 @@ unique: sk->sk_hash = hash; BUG_TRAP(sk_unhashed(sk)); __sk_add_node(sk, &head->chain); - sock_prot_inc_use(sk->sk_prot); + sock_prot_inuse_add(sk->sk_prot, 1); write_unlock(lock); if (twp) { @@ -267,22 +316,102 @@ static inline u32 inet_sk_port_offset(const struct sock *sk) inet->dport); } -/* - * Bind a port for a connect operation and hash it. - */ -int inet_hash_connect(struct inet_timewait_death_row *death_row, - struct sock *sk) +void __inet_hash_nolisten(struct sock *sk) +{ + struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; + struct hlist_head *list; + rwlock_t *lock; + struct inet_ehash_bucket *head; + + BUG_TRAP(sk_unhashed(sk)); + + sk->sk_hash = inet_sk_ehashfn(sk); + head = inet_ehash_bucket(hashinfo, sk->sk_hash); + list = &head->chain; + lock = inet_ehash_lockp(hashinfo, sk->sk_hash); + + write_lock(lock); + __sk_add_node(sk, list); + sock_prot_inuse_add(sk->sk_prot, 1); + write_unlock(lock); +} +EXPORT_SYMBOL_GPL(__inet_hash_nolisten); + +static void __inet_hash(struct sock *sk) +{ + struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; + struct hlist_head *list; + rwlock_t *lock; + + if (sk->sk_state != TCP_LISTEN) { + __inet_hash_nolisten(sk); + return; + } + + BUG_TRAP(sk_unhashed(sk)); + list = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)]; + lock = &hashinfo->lhash_lock; + + inet_listen_wlock(hashinfo); + __sk_add_node(sk, list); + sock_prot_inuse_add(sk->sk_prot, 1); + write_unlock(lock); + wake_up(&hashinfo->lhash_wait); +} + +void inet_hash(struct sock *sk) +{ + if (sk->sk_state != TCP_CLOSE) { + local_bh_disable(); + __inet_hash(sk); + local_bh_enable(); + } +} +EXPORT_SYMBOL_GPL(inet_hash); + +void inet_unhash(struct sock *sk) +{ + rwlock_t *lock; + struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; + + if (sk_unhashed(sk)) + goto out; + + if (sk->sk_state == TCP_LISTEN) { + local_bh_disable(); + inet_listen_wlock(hashinfo); + lock = &hashinfo->lhash_lock; + } else { + lock = inet_ehash_lockp(hashinfo, sk->sk_hash); + write_lock_bh(lock); + } + + if (__sk_del_node_init(sk)) + sock_prot_inuse_add(sk->sk_prot, -1); + write_unlock_bh(lock); +out: + if (sk->sk_state == TCP_LISTEN) + wake_up(&hashinfo->lhash_wait); +} +EXPORT_SYMBOL_GPL(inet_unhash); + +int __inet_hash_connect(struct inet_timewait_death_row *death_row, + struct sock *sk, u32 port_offset, + int (*check_established)(struct inet_timewait_death_row *, + struct sock *, __u16, struct inet_timewait_sock **), + void (*hash)(struct sock *sk)) { struct inet_hashinfo *hinfo = death_row->hashinfo; const unsigned short snum = inet_sk(sk)->num; struct inet_bind_hashbucket *head; struct inet_bind_bucket *tb; int ret; + struct net *net = sk->sk_net; if (!snum) { int i, remaining, low, high, port; static u32 hint; - u32 offset = hint + inet_sk_port_offset(sk); + u32 offset = hint + port_offset; struct hlist_node *node; struct inet_timewait_sock *tw = NULL; @@ -300,19 +429,19 @@ int inet_hash_connect(struct inet_timewait_death_row *death_row, * unique enough. */ inet_bind_bucket_for_each(tb, node, &head->chain) { - if (tb->port == port) { + if (tb->ib_net == net && tb->port == port) { BUG_TRAP(!hlist_empty(&tb->owners)); if (tb->fastreuse >= 0) goto next_port; - if (!__inet_check_established(death_row, - sk, port, - &tw)) + if (!check_established(death_row, sk, + port, &tw)) goto ok; goto next_port; } } - tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, head, port); + tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, + net, head, port); if (!tb) { spin_unlock(&head->lock); break; @@ -334,7 +463,7 @@ ok: inet_bind_hash(sk, tb, port); if (sk_unhashed(sk)) { inet_sk(sk)->sport = htons(port); - __inet_hash(hinfo, sk, 0); + hash(sk); } spin_unlock(&head->lock); @@ -351,17 +480,27 @@ ok: tb = inet_csk(sk)->icsk_bind_hash; spin_lock_bh(&head->lock); if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) { - __inet_hash(hinfo, sk, 0); + hash(sk); spin_unlock_bh(&head->lock); return 0; } else { spin_unlock(&head->lock); /* No definite answer... Walk to established hash table */ - ret = __inet_check_established(death_row, sk, snum, NULL); + ret = check_established(death_row, sk, snum, NULL); out: local_bh_enable(); return ret; } } +/* + * Bind a port for a connect operation and hash it. + */ +int inet_hash_connect(struct inet_timewait_death_row *death_row, + struct sock *sk) +{ + return __inet_hash_connect(death_row, sk, inet_sk_port_offset(sk), + __inet_check_established, __inet_hash_nolisten); +} + EXPORT_SYMBOL_GPL(inet_hash_connect);