Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-10-30 08:42:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'rhashtable'

Browse Source 
Daniel Borkmann says:

====================
rhashtable updates

As discussed, I'm sending out rhashtable fixups for -net.

I have a couple of more patches I was working on last week pending,
i.e. to get rid of ht->nelems and ht->shift atomic operations which
speed-up pure insertions/deletions, e.g. on my laptop I have 2 threads,
inserting 7M entries each, that will reduce insertion time from ~1,450 ms
to 865 ms (performance should even be better after removing the
grow/shrink indirections). I guess that however is rather something
for net-next.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller 2015-02-27 16:06:21 -05:00
commit c0eebfa323
6 changed files with 19 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
include/linux/rhashtable.h
View File

@ -79,12 +79,6 @@ struct rhashtable;
* @locks_mul: Number of bucket locks to allocate per cpu (default: 128)
* @hashfn: Function to hash key
* @obj_hashfn: Function to hash object
* @grow_decision: If defined, may return true if table should expand
* @shrink_decision: If defined, may return true if table should shrink
*
* Note: when implementing the grow and shrink decision function, min/max
* shift must be enforced, otherwise, resizing watermarks they set may be
* useless.
*/
struct rhashtable_params {
size_t nelem_hint;
@ -98,10 +92,6 @@ struct rhashtable_params {
size_t locks_mul;
rht_hashfn_t hashfn;
rht_obj_hashfn_t obj_hashfn;
bool (*grow_decision)(const struct rhashtable *ht,
size_t new_size);
bool (*shrink_decision)(const struct rhashtable *ht,
size_t new_size);
};
/**
@ -193,9 +183,6 @@ int rhashtable_init(struct rhashtable *ht, struct rhashtable_params *params);
void rhashtable_insert(struct rhashtable *ht, struct rhash_head *node);
bool rhashtable_remove(struct rhashtable *ht, struct rhash_head *node);
bool rht_grow_above_75(const struct rhashtable *ht, size_t new_size);
bool rht_shrink_below_30(const struct rhashtable *ht, size_t new_size);
int rhashtable_expand(struct rhashtable *ht);
int rhashtable_shrink(struct rhashtable *ht);

58
lib/rhashtable.c
View File

@ -247,26 +247,24 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
* @ht: hash table
* @new_size: new table size
*/
bool rht_grow_above_75(const struct rhashtable *ht, size_t new_size)
static bool rht_grow_above_75(const struct rhashtable *ht, size_t new_size)
{
/* Expand table when exceeding 75% load */
return atomic_read(&ht->nelems) > (new_size / 4 * 3) &&
(ht->p.max_shift && atomic_read(&ht->shift) < ht->p.max_shift);
(!ht->p.max_shift || atomic_read(&ht->shift) < ht->p.max_shift);
}
EXPORT_SYMBOL_GPL(rht_grow_above_75);
/**
* rht_shrink_below_30 - returns true if nelems < 0.3 * table-size
* @ht: hash table
* @new_size: new table size
*/
bool rht_shrink_below_30(const struct rhashtable *ht, size_t new_size)
static bool rht_shrink_below_30(const struct rhashtable *ht, size_t new_size)
{
/* Shrink table beneath 30% load */
return atomic_read(&ht->nelems) < (new_size * 3 / 10) &&
(atomic_read(&ht->shift) > ht->p.min_shift);
}
EXPORT_SYMBOL_GPL(rht_shrink_below_30);
static void lock_buckets(struct bucket_table *new_tbl,
struct bucket_table *old_tbl, unsigned int hash)
@ -528,40 +526,19 @@ static void rht_deferred_worker(struct work_struct *work)
list_for_each_entry(walker, &ht->walkers, list)
walker->resize = true;
if (ht->p.grow_decision && ht->p.grow_decision(ht, tbl->size))
if (rht_grow_above_75(ht, tbl->size))
rhashtable_expand(ht);
else if (ht->p.shrink_decision && ht->p.shrink_decision(ht, tbl->size))
else if (rht_shrink_below_30(ht, tbl->size))
rhashtable_shrink(ht);
unlock:
mutex_unlock(&ht->mutex);
}
static void rhashtable_probe_expand(struct rhashtable *ht)
{
const struct bucket_table *new_tbl = rht_dereference_rcu(ht->future_tbl, ht);
const struct bucket_table *tbl = rht_dereference_rcu(ht->tbl, ht);
/* Only adjust the table if no resizing is currently in progress. */
if (tbl == new_tbl && ht->p.grow_decision &&
ht->p.grow_decision(ht, tbl->size))
schedule_work(&ht->run_work);
}
static void rhashtable_probe_shrink(struct rhashtable *ht)
{
const struct bucket_table *new_tbl = rht_dereference_rcu(ht->future_tbl, ht);
const struct bucket_table *tbl = rht_dereference_rcu(ht->tbl, ht);
/* Only adjust the table if no resizing is currently in progress. */
if (tbl == new_tbl && ht->p.shrink_decision &&
ht->p.shrink_decision(ht, tbl->size))
schedule_work(&ht->run_work);
}
static void __rhashtable_insert(struct rhashtable *ht, struct rhash_head *obj,
struct bucket_table *tbl, u32 hash)
struct bucket_table *tbl,
const struct bucket_table *old_tbl, u32 hash)
{
bool no_resize_running = tbl == old_tbl;
struct rhash_head *head;
hash = rht_bucket_index(tbl, hash);
@ -577,8 +554,8 @@ static void __rhashtable_insert(struct rhashtable *ht, struct rhash_head *obj,
rcu_assign_pointer(tbl->buckets[hash], obj);
atomic_inc(&ht->nelems);
rhashtable_probe_expand(ht);
if (no_resize_running && rht_grow_above_75(ht, tbl->size))
schedule_work(&ht->run_work);
}
/**
@ -608,7 +585,7 @@ void rhashtable_insert(struct rhashtable *ht, struct rhash_head *obj)
hash = obj_raw_hashfn(ht, rht_obj(ht, obj));
lock_buckets(tbl, old_tbl, hash);
__rhashtable_insert(ht, obj, tbl, hash);
__rhashtable_insert(ht, obj, tbl, old_tbl, hash);
unlock_buckets(tbl, old_tbl, hash);
rcu_read_unlock();
@ -690,8 +667,11 @@ found:
unlock_buckets(new_tbl, old_tbl, new_hash);
if (ret) {
bool no_resize_running = new_tbl == old_tbl;
atomic_dec(&ht->nelems);
rhashtable_probe_shrink(ht);
if (no_resize_running && rht_shrink_below_30(ht, new_tbl->size))
schedule_work(&ht->run_work);
}
rcu_read_unlock();
@ -861,7 +841,7 @@ bool rhashtable_lookup_compare_insert(struct rhashtable *ht,
goto exit;
}
__rhashtable_insert(ht, obj, new_tbl, new_hash);
__rhashtable_insert(ht, obj, new_tbl, old_tbl, new_hash);
exit:
unlock_buckets(new_tbl, old_tbl, new_hash);
@ -1123,8 +1103,7 @@ int rhashtable_init(struct rhashtable *ht, struct rhashtable_params *params)
if (!ht->p.hash_rnd)
get_random_bytes(&ht->p.hash_rnd, sizeof(ht->p.hash_rnd));
if (ht->p.grow_decision || ht->p.shrink_decision)
INIT_WORK(&ht->run_work, rht_deferred_worker);
INIT_WORK(&ht->run_work, rht_deferred_worker);
return 0;
}
@ -1142,8 +1121,7 @@ void rhashtable_destroy(struct rhashtable *ht)
{
ht->being_destroyed = true;
if (ht->p.grow_decision || ht->p.shrink_decision)
cancel_work_sync(&ht->run_work);
cancel_work_sync(&ht->run_work);
mutex_lock(&ht->mutex);
bucket_table_free(rht_dereference(ht->tbl, ht));

3
lib/test_rhashtable.c
View File

@ -201,9 +201,8 @@ static int __init test_rht_init(void)
.key_offset = offsetof(struct test_obj, value),
.key_len = sizeof(int),
.hashfn = jhash,
.max_shift = 1, /* we expand/shrink manually here */
.nulls_base = (3U << RHT_BASE_SHIFT),
.grow_decision = rht_grow_above_75,
.shrink_decision = rht_shrink_below_30,
};
int err;

2
net/netfilter/nft_hash.c
View File

@ -192,8 +192,6 @@ static int nft_hash_init(const struct nft_set *set,
.key_offset = offsetof(struct nft_hash_elem, key),
.key_len = set->klen,
.hashfn = jhash,
.grow_decision = rht_grow_above_75,
.shrink_decision = rht_shrink_below_30,
};
return rhashtable_init(priv, &params);

2
net/netlink/af_netlink.c
View File

@ -3126,8 +3126,6 @@ static int __init netlink_proto_init(void)
.key_len = sizeof(u32), /* portid */
.hashfn = jhash,
.max_shift = 16, /* 64K */
.grow_decision = rht_grow_above_75,
.shrink_decision = rht_shrink_below_30,
};
if (err != 0)

2
net/tipc/socket.c
View File

@ -2364,8 +2364,6 @@ int tipc_sk_rht_init(struct net *net)
.hashfn = jhash,
.max_shift = 20, /* 1M */
.min_shift = 8, /* 256 */
.grow_decision = rht_grow_above_75,
.shrink_decision = rht_shrink_below_30,
};
return rhashtable_init(&tn->sk_rht, &rht_params);