summaryrefslogtreecommitdiffstats
path: root/fs/dcache.c
diff options
context:
space:
mode:
Diffstat (limited to 'fs/dcache.c')
-rw-r--r--fs/dcache.c88
1 files changed, 33 insertions, 55 deletions
diff --git a/fs/dcache.c b/fs/dcache.c
index 129a3573099..18b2a1f10ed 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -35,6 +35,7 @@
#include <linux/hardirq.h>
#include <linux/bit_spinlock.h>
#include <linux/rculist_bl.h>
+#include <linux/prefetch.h>
#include "internal.h"
/*
@@ -99,12 +100,9 @@ static struct kmem_cache *dentry_cache __read_mostly;
static unsigned int d_hash_mask __read_mostly;
static unsigned int d_hash_shift __read_mostly;
-struct dcache_hash_bucket {
- struct hlist_bl_head head;
-};
-static struct dcache_hash_bucket *dentry_hashtable __read_mostly;
+static struct hlist_bl_head *dentry_hashtable __read_mostly;
-static inline struct dcache_hash_bucket *d_hash(struct dentry *parent,
+static inline struct hlist_bl_head *d_hash(struct dentry *parent,
unsigned long hash)
{
hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES;
@@ -112,16 +110,6 @@ static inline struct dcache_hash_bucket *d_hash(struct dentry *parent,
return dentry_hashtable + (hash & D_HASHMASK);
}
-static inline void spin_lock_bucket(struct dcache_hash_bucket *b)
-{
- bit_spin_lock(0, (unsigned long *)&b->head.first);
-}
-
-static inline void spin_unlock_bucket(struct dcache_hash_bucket *b)
-{
- __bit_spin_unlock(0, (unsigned long *)&b->head.first);
-}
-
/* Statistics gathering. */
struct dentry_stat_t dentry_stat = {
.age_limit = 45,
@@ -167,8 +155,8 @@ static void d_free(struct dentry *dentry)
if (dentry->d_op && dentry->d_op->d_release)
dentry->d_op->d_release(dentry);
- /* if dentry was never inserted into hash, immediate free is OK */
- if (hlist_bl_unhashed(&dentry->d_hash))
+ /* if dentry was never visible to RCU, immediate free is OK */
+ if (!(dentry->d_flags & DCACHE_RCUACCESS))
__d_free(&dentry->d_u.d_rcu);
else
call_rcu(&dentry->d_u.d_rcu, __d_free);
@@ -330,28 +318,19 @@ static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)
*/
void __d_drop(struct dentry *dentry)
{
- if (!(dentry->d_flags & DCACHE_UNHASHED)) {
- if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED)) {
- bit_spin_lock(0,
- (unsigned long *)&dentry->d_sb->s_anon.first);
- dentry->d_flags |= DCACHE_UNHASHED;
- hlist_bl_del_init(&dentry->d_hash);
- __bit_spin_unlock(0,
- (unsigned long *)&dentry->d_sb->s_anon.first);
- } else {
- struct dcache_hash_bucket *b;
+ if (!d_unhashed(dentry)) {
+ struct hlist_bl_head *b;
+ if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED))
+ b = &dentry->d_sb->s_anon;
+ else
b = d_hash(dentry->d_parent, dentry->d_name.hash);
- spin_lock_bucket(b);
- /*
- * We may not actually need to put DCACHE_UNHASHED
- * manipulations under the hash lock, but follow
- * the principle of least surprise.
- */
- dentry->d_flags |= DCACHE_UNHASHED;
- hlist_bl_del_rcu(&dentry->d_hash);
- spin_unlock_bucket(b);
- dentry_rcuwalk_barrier(dentry);
- }
+
+ hlist_bl_lock(b);
+ __hlist_bl_del(&dentry->d_hash);
+ dentry->d_hash.pprev = NULL;
+ hlist_bl_unlock(b);
+
+ dentry_rcuwalk_barrier(dentry);
}
}
EXPORT_SYMBOL(__d_drop);
@@ -1304,7 +1283,7 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
dname[name->len] = 0;
dentry->d_count = 1;
- dentry->d_flags = DCACHE_UNHASHED;
+ dentry->d_flags = 0;
spin_lock_init(&dentry->d_lock);
seqcount_init(&dentry->d_seq);
dentry->d_inode = NULL;
@@ -1606,10 +1585,9 @@ struct dentry *d_obtain_alias(struct inode *inode)
tmp->d_inode = inode;
tmp->d_flags |= DCACHE_DISCONNECTED;
list_add(&tmp->d_alias, &inode->i_dentry);
- bit_spin_lock(0, (unsigned long *)&tmp->d_sb->s_anon.first);
- tmp->d_flags &= ~DCACHE_UNHASHED;
+ hlist_bl_lock(&tmp->d_sb->s_anon);
hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
- __bit_spin_unlock(0, (unsigned long *)&tmp->d_sb->s_anon.first);
+ hlist_bl_unlock(&tmp->d_sb->s_anon);
spin_unlock(&tmp->d_lock);
spin_unlock(&inode->i_lock);
security_d_instantiate(tmp, inode);
@@ -1789,7 +1767,7 @@ struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name,
unsigned int len = name->len;
unsigned int hash = name->hash;
const unsigned char *str = name->name;
- struct dcache_hash_bucket *b = d_hash(parent, hash);
+ struct hlist_bl_head *b = d_hash(parent, hash);
struct hlist_bl_node *node;
struct dentry *dentry;
@@ -1813,7 +1791,7 @@ struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name,
*
* See Documentation/filesystems/path-lookup.txt for more details.
*/
- hlist_bl_for_each_entry_rcu(dentry, node, &b->head, d_hash) {
+ hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
struct inode *i;
const char *tname;
int tlen;
@@ -1908,7 +1886,7 @@ struct dentry *__d_lookup(struct dentry *parent, struct qstr *name)
unsigned int len = name->len;
unsigned int hash = name->hash;
const unsigned char *str = name->name;
- struct dcache_hash_bucket *b = d_hash(parent, hash);
+ struct hlist_bl_head *b = d_hash(parent, hash);
struct hlist_bl_node *node;
struct dentry *found = NULL;
struct dentry *dentry;
@@ -1935,7 +1913,7 @@ struct dentry *__d_lookup(struct dentry *parent, struct qstr *name)
*/
rcu_read_lock();
- hlist_bl_for_each_entry_rcu(dentry, node, &b->head, d_hash) {
+ hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
const char *tname;
int tlen;
@@ -2086,13 +2064,13 @@ again:
}
EXPORT_SYMBOL(d_delete);
-static void __d_rehash(struct dentry * entry, struct dcache_hash_bucket *b)
+static void __d_rehash(struct dentry * entry, struct hlist_bl_head *b)
{
BUG_ON(!d_unhashed(entry));
- spin_lock_bucket(b);
- entry->d_flags &= ~DCACHE_UNHASHED;
- hlist_bl_add_head_rcu(&entry->d_hash, &b->head);
- spin_unlock_bucket(b);
+ hlist_bl_lock(b);
+ entry->d_flags |= DCACHE_RCUACCESS;
+ hlist_bl_add_head_rcu(&entry->d_hash, b);
+ hlist_bl_unlock(b);
}
static void _d_rehash(struct dentry * entry)
@@ -3025,7 +3003,7 @@ static void __init dcache_init_early(void)
dentry_hashtable =
alloc_large_system_hash("Dentry cache",
- sizeof(struct dcache_hash_bucket),
+ sizeof(struct hlist_bl_head),
dhash_entries,
13,
HASH_EARLY,
@@ -3034,7 +3012,7 @@ static void __init dcache_init_early(void)
0);
for (loop = 0; loop < (1 << d_hash_shift); loop++)
- INIT_HLIST_BL_HEAD(&dentry_hashtable[loop].head);
+ INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
}
static void __init dcache_init(void)
@@ -3057,7 +3035,7 @@ static void __init dcache_init(void)
dentry_hashtable =
alloc_large_system_hash("Dentry cache",
- sizeof(struct dcache_hash_bucket),
+ sizeof(struct hlist_bl_head),
dhash_entries,
13,
0,
@@ -3066,7 +3044,7 @@ static void __init dcache_init(void)
0);
for (loop = 0; loop < (1 << d_hash_shift); loop++)
- INIT_HLIST_BL_HEAD(&dentry_hashtable[loop].head);
+ INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
}
/* SLAB cache for __getname() consumers */
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-12-22 17:28:54 +0000