Loading...
1// SPDX-License-Identifier: GPL-2.0
2#include <stdlib.h>
3#include <assert.h>
4#include <stdio.h>
5#include <linux/types.h>
6#include <linux/kernel.h>
7#include <linux/bitops.h>
8
9#include "test.h"
10
11struct item *
12item_tag_set(struct radix_tree_root *root, unsigned long index, int tag)
13{
14 return radix_tree_tag_set(root, index, tag);
15}
16
17struct item *
18item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag)
19{
20 return radix_tree_tag_clear(root, index, tag);
21}
22
23int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag)
24{
25 return radix_tree_tag_get(root, index, tag);
26}
27
28int __item_insert(struct radix_tree_root *root, struct item *item)
29{
30 return __radix_tree_insert(root, item->index, item->order, item);
31}
32
33struct item *item_create(unsigned long index, unsigned int order)
34{
35 struct item *ret = malloc(sizeof(*ret));
36
37 ret->index = index;
38 ret->order = order;
39 return ret;
40}
41
42int item_insert_order(struct radix_tree_root *root, unsigned long index,
43 unsigned order)
44{
45 struct item *item = item_create(index, order);
46 int err = __item_insert(root, item);
47 if (err)
48 free(item);
49 return err;
50}
51
52int item_insert(struct radix_tree_root *root, unsigned long index)
53{
54 return item_insert_order(root, index, 0);
55}
56
57void item_sanity(struct item *item, unsigned long index)
58{
59 unsigned long mask;
60 assert(!radix_tree_is_internal_node(item));
61 assert(item->order < BITS_PER_LONG);
62 mask = (1UL << item->order) - 1;
63 assert((item->index | mask) == (index | mask));
64}
65
66int item_delete(struct radix_tree_root *root, unsigned long index)
67{
68 struct item *item = radix_tree_delete(root, index);
69
70 if (item) {
71 item_sanity(item, index);
72 free(item);
73 return 1;
74 }
75 return 0;
76}
77
78static void item_free_rcu(struct rcu_head *head)
79{
80 struct item *item = container_of(head, struct item, rcu_head);
81
82 free(item);
83}
84
85int item_delete_rcu(struct radix_tree_root *root, unsigned long index)
86{
87 struct item *item = radix_tree_delete(root, index);
88
89 if (item) {
90 item_sanity(item, index);
91 call_rcu(&item->rcu_head, item_free_rcu);
92 return 1;
93 }
94 return 0;
95}
96
97void item_check_present(struct radix_tree_root *root, unsigned long index)
98{
99 struct item *item;
100
101 item = radix_tree_lookup(root, index);
102 assert(item != NULL);
103 item_sanity(item, index);
104}
105
106struct item *item_lookup(struct radix_tree_root *root, unsigned long index)
107{
108 return radix_tree_lookup(root, index);
109}
110
111void item_check_absent(struct radix_tree_root *root, unsigned long index)
112{
113 struct item *item;
114
115 item = radix_tree_lookup(root, index);
116 assert(item == NULL);
117}
118
119/*
120 * Scan only the passed (start, start+nr] for present items
121 */
122void item_gang_check_present(struct radix_tree_root *root,
123 unsigned long start, unsigned long nr,
124 int chunk, int hop)
125{
126 struct item *items[chunk];
127 unsigned long into;
128
129 for (into = 0; into < nr; ) {
130 int nfound;
131 int nr_to_find = chunk;
132 int i;
133
134 if (nr_to_find > (nr - into))
135 nr_to_find = nr - into;
136
137 nfound = radix_tree_gang_lookup(root, (void **)items,
138 start + into, nr_to_find);
139 assert(nfound == nr_to_find);
140 for (i = 0; i < nfound; i++)
141 assert(items[i]->index == start + into + i);
142 into += hop;
143 }
144}
145
146/*
147 * Scan the entire tree, only expecting present items (start, start+nr]
148 */
149void item_full_scan(struct radix_tree_root *root, unsigned long start,
150 unsigned long nr, int chunk)
151{
152 struct item *items[chunk];
153 unsigned long into = 0;
154 unsigned long this_index = start;
155 int nfound;
156 int i;
157
158// printf("%s(0x%08lx, 0x%08lx, %d)\n", __FUNCTION__, start, nr, chunk);
159
160 while ((nfound = radix_tree_gang_lookup(root, (void **)items, into,
161 chunk))) {
162// printf("At 0x%08lx, nfound=%d\n", into, nfound);
163 for (i = 0; i < nfound; i++) {
164 assert(items[i]->index == this_index);
165 this_index++;
166 }
167// printf("Found 0x%08lx->0x%08lx\n",
168// items[0]->index, items[nfound-1]->index);
169 into = this_index;
170 }
171 if (chunk)
172 assert(this_index == start + nr);
173 nfound = radix_tree_gang_lookup(root, (void **)items,
174 this_index, chunk);
175 assert(nfound == 0);
176}
177
178/* Use the same pattern as tag_pages_for_writeback() in mm/page-writeback.c */
179int tag_tagged_items(struct radix_tree_root *root, pthread_mutex_t *lock,
180 unsigned long start, unsigned long end, unsigned batch,
181 unsigned iftag, unsigned thentag)
182{
183 unsigned long tagged = 0;
184 struct radix_tree_iter iter;
185 void **slot;
186
187 if (batch == 0)
188 batch = 1;
189
190 if (lock)
191 pthread_mutex_lock(lock);
192 radix_tree_for_each_tagged(slot, root, &iter, start, iftag) {
193 if (iter.index > end)
194 break;
195 radix_tree_iter_tag_set(root, &iter, thentag);
196 tagged++;
197 if ((tagged % batch) != 0)
198 continue;
199 slot = radix_tree_iter_resume(slot, &iter);
200 if (lock) {
201 pthread_mutex_unlock(lock);
202 rcu_barrier();
203 pthread_mutex_lock(lock);
204 }
205 }
206 if (lock)
207 pthread_mutex_unlock(lock);
208
209 return tagged;
210}
211
212/* Use the same pattern as find_swap_entry() in mm/shmem.c */
213unsigned long find_item(struct radix_tree_root *root, void *item)
214{
215 struct radix_tree_iter iter;
216 void **slot;
217 unsigned long found = -1;
218 unsigned long checked = 0;
219
220 radix_tree_for_each_slot(slot, root, &iter, 0) {
221 if (*slot == item) {
222 found = iter.index;
223 break;
224 }
225 checked++;
226 if ((checked % 4) != 0)
227 continue;
228 slot = radix_tree_iter_resume(slot, &iter);
229 }
230
231 return found;
232}
233
234static int verify_node(struct radix_tree_node *slot, unsigned int tag,
235 int tagged)
236{
237 int anyset = 0;
238 int i;
239 int j;
240
241 slot = entry_to_node(slot);
242
243 /* Verify consistency at this level */
244 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) {
245 if (slot->tags[tag][i]) {
246 anyset = 1;
247 break;
248 }
249 }
250 if (tagged != anyset) {
251 printf("tag: %u, shift %u, tagged: %d, anyset: %d\n",
252 tag, slot->shift, tagged, anyset);
253 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
254 printf("tag %d: ", j);
255 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
256 printf("%016lx ", slot->tags[j][i]);
257 printf("\n");
258 }
259 return 1;
260 }
261 assert(tagged == anyset);
262
263 /* Go for next level */
264 if (slot->shift > 0) {
265 for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
266 if (slot->slots[i])
267 if (verify_node(slot->slots[i], tag,
268 !!test_bit(i, slot->tags[tag]))) {
269 printf("Failure at off %d\n", i);
270 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
271 printf("tag %d: ", j);
272 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
273 printf("%016lx ", slot->tags[j][i]);
274 printf("\n");
275 }
276 return 1;
277 }
278 }
279 return 0;
280}
281
282void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
283{
284 struct radix_tree_node *node = root->rnode;
285 if (!radix_tree_is_internal_node(node))
286 return;
287 verify_node(node, tag, !!root_tag_get(root, tag));
288}
289
290void item_kill_tree(struct radix_tree_root *root)
291{
292 struct radix_tree_iter iter;
293 void **slot;
294 struct item *items[32];
295 int nfound;
296
297 radix_tree_for_each_slot(slot, root, &iter, 0) {
298 if (radix_tree_exceptional_entry(*slot))
299 radix_tree_delete(root, iter.index);
300 }
301
302 while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) {
303 int i;
304
305 for (i = 0; i < nfound; i++) {
306 void *ret;
307
308 ret = radix_tree_delete(root, items[i]->index);
309 assert(ret == items[i]);
310 free(items[i]);
311 }
312 }
313 assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0);
314 assert(root->rnode == NULL);
315}
316
317void tree_verify_min_height(struct radix_tree_root *root, int maxindex)
318{
319 unsigned shift;
320 struct radix_tree_node *node = root->rnode;
321 if (!radix_tree_is_internal_node(node)) {
322 assert(maxindex == 0);
323 return;
324 }
325
326 node = entry_to_node(node);
327 assert(maxindex <= node_maxindex(node));
328
329 shift = node->shift;
330 if (shift > 0)
331 assert(maxindex > shift_maxindex(shift - RADIX_TREE_MAP_SHIFT));
332 else
333 assert(maxindex > 0);
334}
1#include <stdlib.h>
2#include <assert.h>
3#include <stdio.h>
4#include <linux/types.h>
5#include <linux/kernel.h>
6#include <linux/bitops.h>
7
8#include "test.h"
9
10struct item *
11item_tag_set(struct radix_tree_root *root, unsigned long index, int tag)
12{
13 return radix_tree_tag_set(root, index, tag);
14}
15
16struct item *
17item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag)
18{
19 return radix_tree_tag_clear(root, index, tag);
20}
21
22int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag)
23{
24 return radix_tree_tag_get(root, index, tag);
25}
26
27int __item_insert(struct radix_tree_root *root, struct item *item)
28{
29 return radix_tree_insert(root, item->index, item);
30}
31
32int item_insert(struct radix_tree_root *root, unsigned long index)
33{
34 return __item_insert(root, item_create(index));
35}
36
37int item_delete(struct radix_tree_root *root, unsigned long index)
38{
39 struct item *item = radix_tree_delete(root, index);
40
41 if (item) {
42 assert(item->index == index);
43 free(item);
44 return 1;
45 }
46 return 0;
47}
48
49struct item *item_create(unsigned long index)
50{
51 struct item *ret = malloc(sizeof(*ret));
52
53 ret->index = index;
54 return ret;
55}
56
57void item_check_present(struct radix_tree_root *root, unsigned long index)
58{
59 struct item *item;
60
61 item = radix_tree_lookup(root, index);
62 assert(item != 0);
63 assert(item->index == index);
64}
65
66struct item *item_lookup(struct radix_tree_root *root, unsigned long index)
67{
68 return radix_tree_lookup(root, index);
69}
70
71void item_check_absent(struct radix_tree_root *root, unsigned long index)
72{
73 struct item *item;
74
75 item = radix_tree_lookup(root, index);
76 assert(item == 0);
77}
78
79/*
80 * Scan only the passed (start, start+nr] for present items
81 */
82void item_gang_check_present(struct radix_tree_root *root,
83 unsigned long start, unsigned long nr,
84 int chunk, int hop)
85{
86 struct item *items[chunk];
87 unsigned long into;
88
89 for (into = 0; into < nr; ) {
90 int nfound;
91 int nr_to_find = chunk;
92 int i;
93
94 if (nr_to_find > (nr - into))
95 nr_to_find = nr - into;
96
97 nfound = radix_tree_gang_lookup(root, (void **)items,
98 start + into, nr_to_find);
99 assert(nfound == nr_to_find);
100 for (i = 0; i < nfound; i++)
101 assert(items[i]->index == start + into + i);
102 into += hop;
103 }
104}
105
106/*
107 * Scan the entire tree, only expecting present items (start, start+nr]
108 */
109void item_full_scan(struct radix_tree_root *root, unsigned long start,
110 unsigned long nr, int chunk)
111{
112 struct item *items[chunk];
113 unsigned long into = 0;
114 unsigned long this_index = start;
115 int nfound;
116 int i;
117
118// printf("%s(0x%08lx, 0x%08lx, %d)\n", __FUNCTION__, start, nr, chunk);
119
120 while ((nfound = radix_tree_gang_lookup(root, (void **)items, into,
121 chunk))) {
122// printf("At 0x%08lx, nfound=%d\n", into, nfound);
123 for (i = 0; i < nfound; i++) {
124 assert(items[i]->index == this_index);
125 this_index++;
126 }
127// printf("Found 0x%08lx->0x%08lx\n",
128// items[0]->index, items[nfound-1]->index);
129 into = this_index;
130 }
131 if (chunk)
132 assert(this_index == start + nr);
133 nfound = radix_tree_gang_lookup(root, (void **)items,
134 this_index, chunk);
135 assert(nfound == 0);
136}
137
138static int verify_node(struct radix_tree_node *slot, unsigned int tag,
139 unsigned int height, int tagged)
140{
141 int anyset = 0;
142 int i;
143 int j;
144
145 slot = indirect_to_ptr(slot);
146
147 /* Verify consistency at this level */
148 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) {
149 if (slot->tags[tag][i]) {
150 anyset = 1;
151 break;
152 }
153 }
154 if (tagged != anyset) {
155 printf("tag: %u, height %u, tagged: %d, anyset: %d\n", tag, height, tagged, anyset);
156 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
157 printf("tag %d: ", j);
158 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
159 printf("%016lx ", slot->tags[j][i]);
160 printf("\n");
161 }
162 return 1;
163 }
164 assert(tagged == anyset);
165
166 /* Go for next level */
167 if (height > 1) {
168 for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
169 if (slot->slots[i])
170 if (verify_node(slot->slots[i], tag, height - 1,
171 !!test_bit(i, slot->tags[tag]))) {
172 printf("Failure at off %d\n", i);
173 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
174 printf("tag %d: ", j);
175 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
176 printf("%016lx ", slot->tags[j][i]);
177 printf("\n");
178 }
179 return 1;
180 }
181 }
182 return 0;
183}
184
185void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
186{
187 if (!root->height)
188 return;
189 verify_node(root->rnode, tag, root->height, !!root_tag_get(root, tag));
190}
191
192void item_kill_tree(struct radix_tree_root *root)
193{
194 struct item *items[32];
195 int nfound;
196
197 while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) {
198 int i;
199
200 for (i = 0; i < nfound; i++) {
201 void *ret;
202
203 ret = radix_tree_delete(root, items[i]->index);
204 assert(ret == items[i]);
205 free(items[i]);
206 }
207 }
208 assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0);
209 assert(root->rnode == NULL);
210}
211
212void tree_verify_min_height(struct radix_tree_root *root, int maxindex)
213{
214 assert(radix_tree_maxindex(root->height) >= maxindex);
215 if (root->height > 1)
216 assert(radix_tree_maxindex(root->height-1) < maxindex);
217 else if (root->height == 1)
218 assert(radix_tree_maxindex(root->height-1) <= maxindex);
219}