* Portions Copyright (C) 2001 Christoph Hellwig
* Copyright (C) 2005 SGI, Christoph Lameter
* Copyright (C) 2006 Nick Piggin
+ * Copyright (C) 2012 Konstantin Khlebnikov
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
}
return 0;
}
+
+/**
+ * radix_tree_find_next_bit - find the next set bit in a memory region
+ *
+ * @addr: The address to base the search on
+ * @size: The bitmap size in bits
+ * @offset: The bitnumber to start searching at
+ *
+ * Unrollable variant of find_next_bit() for constant size arrays.
+ * Tail bits starting from size to roundup(size, BITS_PER_LONG) must be zero.
+ * Returns next bit offset, or size if nothing found.
+ */
+static __always_inline unsigned long
+radix_tree_find_next_bit(const unsigned long *addr,
+ unsigned long size, unsigned long offset)
+{
+ if (!__builtin_constant_p(size))
+ return find_next_bit(addr, size, offset);
+
+ if (offset < size) {
+ unsigned long tmp;
+
+ addr += offset / BITS_PER_LONG;
+ tmp = *addr >> (offset % BITS_PER_LONG);
+ if (tmp)
+ return __ffs(tmp) + offset;
+ offset = (offset + BITS_PER_LONG) & ~(BITS_PER_LONG - 1);
+ while (offset < size) {
+ tmp = *++addr;
+ if (tmp)
+ return __ffs(tmp) + offset;
+ offset += BITS_PER_LONG;
+ }
+ }
+ return size;
+}
+
/*
* This assumes that the caller has performed appropriate preallocation, and
* that the caller has pinned this thread of control to the current CPU.
}
EXPORT_SYMBOL(radix_tree_tag_get);
+/**
+ * radix_tree_next_chunk - find next chunk of slots for iteration
+ *
+ * @root: radix tree root
+ * @iter: iterator state
+ * @flags: RADIX_TREE_ITER_* flags and tag index
+ * Returns: pointer to chunk first slot, or NULL if iteration is over
+ */
+void **radix_tree_next_chunk(struct radix_tree_root *root,
+ struct radix_tree_iter *iter, unsigned flags)
+{
+ unsigned shift, tag = flags & RADIX_TREE_ITER_TAG_MASK;
+ struct radix_tree_node *rnode, *node;
+ unsigned long index, offset;
+
+ if ((flags & RADIX_TREE_ITER_TAGGED) && !root_tag_get(root, tag))
+ return NULL;
+
+ /*
+ * Catch next_index overflow after ~0UL. iter->index never overflows
+ * during iterating; it can be zero only at the beginning.
+ * And we cannot overflow iter->next_index in a single step,
+ * because RADIX_TREE_MAP_SHIFT < BITS_PER_LONG.
+ */
+ index = iter->next_index;
+ if (!index && iter->index)
+ return NULL;
+
+ rnode = rcu_dereference_raw(root->rnode);
+ if (radix_tree_is_indirect_ptr(rnode)) {
+ rnode = indirect_to_ptr(rnode);
+ } else if (rnode && !index) {
+ /* Single-slot tree */
+ iter->index = 0;
+ iter->next_index = 1;
+ iter->tags = 1;
+ return (void **)&root->rnode;
+ } else
+ return NULL;
+
+restart:
+ shift = (rnode->height - 1) * RADIX_TREE_MAP_SHIFT;
+ offset = index >> shift;
+
+ /* Index outside of the tree */
+ if (offset >= RADIX_TREE_MAP_SIZE)
+ return NULL;
+
+ node = rnode;
+ while (1) {
+ if ((flags & RADIX_TREE_ITER_TAGGED) ?
+ !test_bit(offset, node->tags[tag]) :
+ !node->slots[offset]) {
+ /* Hole detected */
+ if (flags & RADIX_TREE_ITER_CONTIG)
+ return NULL;
+
+ if (flags & RADIX_TREE_ITER_TAGGED)
+ offset = radix_tree_find_next_bit(
+ node->tags[tag],
+ RADIX_TREE_MAP_SIZE,
+ offset + 1);
+ else
+ while (++offset < RADIX_TREE_MAP_SIZE) {
+ if (node->slots[offset])
+ break;
+ }
+ index &= ~((RADIX_TREE_MAP_SIZE << shift) - 1);
+ index += offset << shift;
+ /* Overflow after ~0UL */
+ if (!index)
+ return NULL;
+ if (offset == RADIX_TREE_MAP_SIZE)
+ goto restart;
+ }
+
+ /* This is leaf-node */
+ if (!shift)
+ break;
+
+ node = rcu_dereference_raw(node->slots[offset]);
+ if (node == NULL)
+ goto restart;
+ shift -= RADIX_TREE_MAP_SHIFT;
+ offset = (index >> shift) & RADIX_TREE_MAP_MASK;
+ }
+
+ /* Update the iterator state */
+ iter->index = index;
+ iter->next_index = (index | RADIX_TREE_MAP_MASK) + 1;
+
+ /* Construct iter->tags bit-mask from node->tags[tag] array */
+ if (flags & RADIX_TREE_ITER_TAGGED) {
+ unsigned tag_long, tag_bit;
+
+ tag_long = offset / BITS_PER_LONG;
+ tag_bit = offset % BITS_PER_LONG;
+ iter->tags = node->tags[tag][tag_long] >> tag_bit;
+ /* This never happens if RADIX_TREE_TAG_LONGS == 1 */
+ if (tag_long < RADIX_TREE_TAG_LONGS - 1) {
+ /* Pick tags from next element */
+ if (tag_bit)
+ iter->tags |= node->tags[tag][tag_long + 1] <<
+ (BITS_PER_LONG - tag_bit);
+ /* Clip chunk size, here only BITS_PER_LONG tags */
+ iter->next_index = index + BITS_PER_LONG;
+ }
+ }
+
+ return node->slots + offset;
+}
+EXPORT_SYMBOL(radix_tree_next_chunk);
+
/**
* radix_tree_range_tag_if_tagged - for each item in given range set given
* tag if item has another tag set
}
EXPORT_SYMBOL(radix_tree_prev_hole);
-static unsigned int
-__lookup(struct radix_tree_node *slot, void ***results, unsigned long *indices,
- unsigned long index, unsigned int max_items, unsigned long *next_index)
-{
- unsigned int nr_found = 0;
- unsigned int shift, height;
- unsigned long i;
-
- height = slot->height;
- if (height == 0)
- goto out;
- shift = (height-1) * RADIX_TREE_MAP_SHIFT;
-
- for ( ; height > 1; height--) {
- i = (index >> shift) & RADIX_TREE_MAP_MASK;
- for (;;) {
- if (slot->slots[i] != NULL)
- break;
- index &= ~((1UL << shift) - 1);
- index += 1UL << shift;
- if (index == 0)
- goto out; /* 32-bit wraparound */
- i++;
- if (i == RADIX_TREE_MAP_SIZE)
- goto out;
- }
-
- shift -= RADIX_TREE_MAP_SHIFT;
- slot = rcu_dereference_raw(slot->slots[i]);
- if (slot == NULL)
- goto out;
- }
-
- /* Bottom level: grab some items */
- for (i = index & RADIX_TREE_MAP_MASK; i < RADIX_TREE_MAP_SIZE; i++) {
- if (slot->slots[i]) {
- results[nr_found] = &(slot->slots[i]);
- if (indices)
- indices[nr_found] = index;
- if (++nr_found == max_items) {
- index++;
- goto out;
- }
- }
- index++;
- }
-out:
- *next_index = index;
- return nr_found;
-}
-
/**
* radix_tree_gang_lookup - perform multiple lookup on a radix tree
* @root: radix tree root
radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
unsigned long first_index, unsigned int max_items)
{
- unsigned long max_index;
- struct radix_tree_node *node;
- unsigned long cur_index = first_index;
- unsigned int ret;
+ struct radix_tree_iter iter;
+ void **slot;
+ unsigned int ret = 0;
- node = rcu_dereference_raw(root->rnode);
- if (!node)
+ if (unlikely(!max_items))
return 0;
- if (!radix_tree_is_indirect_ptr(node)) {
- if (first_index > 0)
- return 0;
- results[0] = node;
- return 1;
- }
- node = indirect_to_ptr(node);
-
- max_index = radix_tree_maxindex(node->height);
-
- ret = 0;
- while (ret < max_items) {
- unsigned int nr_found, slots_found, i;
- unsigned long next_index; /* Index of next search */
-
- if (cur_index > max_index)
- break;
- slots_found = __lookup(node, (void ***)results + ret, NULL,
- cur_index, max_items - ret, &next_index);
- nr_found = 0;
- for (i = 0; i < slots_found; i++) {
- struct radix_tree_node *slot;
- slot = *(((void ***)results)[ret + i]);
- if (!slot)
- continue;
- results[ret + nr_found] =
- indirect_to_ptr(rcu_dereference_raw(slot));
- nr_found++;
- }
- ret += nr_found;
- if (next_index == 0)
+ radix_tree_for_each_slot(slot, root, &iter, first_index) {
+ results[ret] = indirect_to_ptr(rcu_dereference_raw(*slot));
+ if (!results[ret])
+ continue;
+ if (++ret == max_items)
break;
- cur_index = next_index;
}
return ret;
void ***results, unsigned long *indices,
unsigned long first_index, unsigned int max_items)
{
- unsigned long max_index;
- struct radix_tree_node *node;
- unsigned long cur_index = first_index;
- unsigned int ret;
+ struct radix_tree_iter iter;
+ void **slot;
+ unsigned int ret = 0;
- node = rcu_dereference_raw(root->rnode);
- if (!node)
+ if (unlikely(!max_items))
return 0;
- if (!radix_tree_is_indirect_ptr(node)) {
- if (first_index > 0)
- return 0;
- results[0] = (void **)&root->rnode;
+ radix_tree_for_each_slot(slot, root, &iter, first_index) {
+ results[ret] = slot;
if (indices)
- indices[0] = 0;
- return 1;
- }
- node = indirect_to_ptr(node);
-
- max_index = radix_tree_maxindex(node->height);
-
- ret = 0;
- while (ret < max_items) {
- unsigned int slots_found;
- unsigned long next_index; /* Index of next search */
-
- if (cur_index > max_index)
+ indices[ret] = iter.index;
+ if (++ret == max_items)
break;
- slots_found = __lookup(node, results + ret,
- indices ? indices + ret : NULL,
- cur_index, max_items - ret, &next_index);
- ret += slots_found;
- if (next_index == 0)
- break;
- cur_index = next_index;
}
return ret;
}
EXPORT_SYMBOL(radix_tree_gang_lookup_slot);
-/*
- * FIXME: the two tag_get()s here should use find_next_bit() instead of
- * open-coding the search.
- */
-static unsigned int
-__lookup_tag(struct radix_tree_node *slot, void ***results, unsigned long index,
- unsigned int max_items, unsigned long *next_index, unsigned int tag)
-{
- unsigned int nr_found = 0;
- unsigned int shift, height;
-
- height = slot->height;
- if (height == 0)
- goto out;
- shift = (height-1) * RADIX_TREE_MAP_SHIFT;
-
- while (height > 0) {
- unsigned long i = (index >> shift) & RADIX_TREE_MAP_MASK ;
-
- for (;;) {
- if (tag_get(slot, tag, i))
- break;
- index &= ~((1UL << shift) - 1);
- index += 1UL << shift;
- if (index == 0)
- goto out; /* 32-bit wraparound */
- i++;
- if (i == RADIX_TREE_MAP_SIZE)
- goto out;
- }
- height--;
- if (height == 0) { /* Bottom level: grab some items */
- unsigned long j = index & RADIX_TREE_MAP_MASK;
-
- for ( ; j < RADIX_TREE_MAP_SIZE; j++) {
- index++;
- if (!tag_get(slot, tag, j))
- continue;
- /*
- * Even though the tag was found set, we need to
- * recheck that we have a non-NULL node, because
- * if this lookup is lockless, it may have been
- * subsequently deleted.
- *
- * Similar care must be taken in any place that
- * lookup ->slots[x] without a lock (ie. can't
- * rely on its value remaining the same).
- */
- if (slot->slots[j]) {
- results[nr_found++] = &(slot->slots[j]);
- if (nr_found == max_items)
- goto out;
- }
- }
- }
- shift -= RADIX_TREE_MAP_SHIFT;
- slot = rcu_dereference_raw(slot->slots[i]);
- if (slot == NULL)
- break;
- }
-out:
- *next_index = index;
- return nr_found;
-}
-
/**
* radix_tree_gang_lookup_tag - perform multiple lookup on a radix tree
* based on a tag
unsigned long first_index, unsigned int max_items,
unsigned int tag)
{
- struct radix_tree_node *node;
- unsigned long max_index;
- unsigned long cur_index = first_index;
- unsigned int ret;
-
- /* check the root's tag bit */
- if (!root_tag_get(root, tag))
- return 0;
+ struct radix_tree_iter iter;
+ void **slot;
+ unsigned int ret = 0;
- node = rcu_dereference_raw(root->rnode);
- if (!node)
+ if (unlikely(!max_items))
return 0;
- if (!radix_tree_is_indirect_ptr(node)) {
- if (first_index > 0)
- return 0;
- results[0] = node;
- return 1;
- }
- node = indirect_to_ptr(node);
-
- max_index = radix_tree_maxindex(node->height);
-
- ret = 0;
- while (ret < max_items) {
- unsigned int nr_found, slots_found, i;
- unsigned long next_index; /* Index of next search */
-
- if (cur_index > max_index)
- break;
- slots_found = __lookup_tag(node, (void ***)results + ret,
- cur_index, max_items - ret, &next_index, tag);
- nr_found = 0;
- for (i = 0; i < slots_found; i++) {
- struct radix_tree_node *slot;
- slot = *(((void ***)results)[ret + i]);
- if (!slot)
- continue;
- results[ret + nr_found] =
- indirect_to_ptr(rcu_dereference_raw(slot));
- nr_found++;
- }
- ret += nr_found;
- if (next_index == 0)
+ radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {
+ results[ret] = indirect_to_ptr(rcu_dereference_raw(*slot));
+ if (!results[ret])
+ continue;
+ if (++ret == max_items)
break;
- cur_index = next_index;
}
return ret;
unsigned long first_index, unsigned int max_items,
unsigned int tag)
{
- struct radix_tree_node *node;
- unsigned long max_index;
- unsigned long cur_index = first_index;
- unsigned int ret;
+ struct radix_tree_iter iter;
+ void **slot;
+ unsigned int ret = 0;
- /* check the root's tag bit */
- if (!root_tag_get(root, tag))
- return 0;
-
- node = rcu_dereference_raw(root->rnode);
- if (!node)
+ if (unlikely(!max_items))
return 0;
- if (!radix_tree_is_indirect_ptr(node)) {
- if (first_index > 0)
- return 0;
- results[0] = (void **)&root->rnode;
- return 1;
- }
- node = indirect_to_ptr(node);
-
- max_index = radix_tree_maxindex(node->height);
-
- ret = 0;
- while (ret < max_items) {
- unsigned int slots_found;
- unsigned long next_index; /* Index of next search */
-
- if (cur_index > max_index)
- break;
- slots_found = __lookup_tag(node, results + ret,
- cur_index, max_items - ret, &next_index, tag);
- ret += slots_found;
- if (next_index == 0)
+ radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {
+ results[ret] = slot;
+ if (++ret == max_items)
break;
- cur_index = next_index;
}
return ret;
projects / karo-tx-linux.git / blobdiff