X-Git-Url: https://git.kernelconcepts.de/?a=blobdiff_plain;f=mm%2Fzsmalloc.c;h=5481c677e4658dd58c74d7a17b30ad1f6f6c5aad;hb=b1b00a5b8a6cf32e3973507decf1216709b55072;hp=839a48c3ca27b2bf008496c89867f7210aeb6610;hpb=911883759fed56c32b97d73154039b5f8ffcba93;p=karo-tx-linux.git diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 839a48c3ca27..5481c677e465 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -214,7 +214,7 @@ struct link_free { }; struct zs_pool { - struct size_class size_class[ZS_SIZE_CLASSES]; + struct size_class *size_class[ZS_SIZE_CLASSES]; gfp_t flags; /* allocation flags used when growing pool */ atomic_long_t pages_allocated; @@ -468,7 +468,7 @@ static enum fullness_group fix_fullness_group(struct zs_pool *pool, if (newfg == currfg) goto out; - class = &pool->size_class[class_idx]; + class = pool->size_class[class_idx]; remove_zspage(page, class, currfg); insert_zspage(page, class, newfg); set_zspage_mapping(page, class_idx, newfg); @@ -881,14 +881,10 @@ static struct notifier_block zs_cpu_nb = { .notifier_call = zs_cpu_notifier }; -static void zs_exit(void) +static void zs_unregister_cpu_notifier(void) { int cpu; -#ifdef CONFIG_ZPOOL - zpool_unregister_driver(&zs_zpool_driver); -#endif - cpu_notifier_register_begin(); for_each_online_cpu(cpu) @@ -898,31 +894,61 @@ static void zs_exit(void) cpu_notifier_register_done(); } -static int zs_init(void) +static int zs_register_cpu_notifier(void) { - int cpu, ret; + int cpu, uninitialized_var(ret); cpu_notifier_register_begin(); __register_cpu_notifier(&zs_cpu_nb); for_each_online_cpu(cpu) { ret = zs_cpu_notifier(NULL, CPU_UP_PREPARE, (void *)(long)cpu); - if (notifier_to_errno(ret)) { - cpu_notifier_register_done(); - goto fail; - } + if (notifier_to_errno(ret)) + break; } cpu_notifier_register_done(); + return notifier_to_errno(ret); +} +static void __exit zs_exit(void) +{ #ifdef CONFIG_ZPOOL - zpool_register_driver(&zs_zpool_driver); + zpool_unregister_driver(&zs_zpool_driver); #endif + zs_unregister_cpu_notifier(); +} + +static int __init zs_init(void) +{ + int ret = zs_register_cpu_notifier(); + if (ret) { + zs_unregister_cpu_notifier(); + return ret; + } + +#ifdef CONFIG_ZPOOL + zpool_register_driver(&zs_zpool_driver); +#endif return 0; -fail: - zs_exit(); - return notifier_to_errno(ret); +} + +static unsigned int get_maxobj_per_zspage(int size, int pages_per_zspage) +{ + return pages_per_zspage * PAGE_SIZE / size; +} + +static bool can_merge(struct size_class *prev, int size, int pages_per_zspage) +{ + if (prev->pages_per_zspage != pages_per_zspage) + return false; + + if (get_maxobj_per_zspage(prev->size, prev->pages_per_zspage) + != get_maxobj_per_zspage(size, pages_per_zspage)) + return false; + + return true; } /** @@ -945,25 +971,56 @@ struct zs_pool *zs_create_pool(gfp_t flags) if (!pool) return NULL; - for (i = 0; i < ZS_SIZE_CLASSES; i++) { + /* + * Iterate reversly, because, size of size_class that we want to use + * for merging should be larger or equal to current size. + */ + for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) { int size; + int pages_per_zspage; struct size_class *class; + struct size_class *prev_class; size = ZS_MIN_ALLOC_SIZE + i * ZS_SIZE_CLASS_DELTA; if (size > ZS_MAX_ALLOC_SIZE) size = ZS_MAX_ALLOC_SIZE; + pages_per_zspage = get_pages_per_zspage(size); + + /* + * size_class is used for normal zsmalloc operation such + * as alloc/free for that size. Although it is natural that we + * have one size_class for each size, there is a chance that we + * can get more memory utilization if we use one size_class for + * many different sizes whose size_class have same + * characteristics. So, we makes size_class point to + * previous size_class if possible. + */ + if (i < ZS_SIZE_CLASSES - 1) { + prev_class = pool->size_class[i + 1]; + if (can_merge(prev_class, size, pages_per_zspage)) { + pool->size_class[i] = prev_class; + continue; + } + } + + class = kzalloc(sizeof(struct size_class), GFP_KERNEL); + if (!class) + goto err; - class = &pool->size_class[i]; class->size = size; class->index = i; + class->pages_per_zspage = pages_per_zspage; spin_lock_init(&class->lock); - class->pages_per_zspage = get_pages_per_zspage(size); - + pool->size_class[i] = class; } pool->flags = flags; return pool; + +err: + zs_destroy_pool(pool); + return NULL; } EXPORT_SYMBOL_GPL(zs_create_pool); @@ -973,7 +1030,13 @@ void zs_destroy_pool(struct zs_pool *pool) for (i = 0; i < ZS_SIZE_CLASSES; i++) { int fg; - struct size_class *class = &pool->size_class[i]; + struct size_class *class = pool->size_class[i]; + + if (!class) + continue; + + if (class->index != i) + continue; for (fg = 0; fg < _ZS_NR_FULLNESS_GROUPS; fg++) { if (class->fullness_list[fg]) { @@ -981,6 +1044,7 @@ void zs_destroy_pool(struct zs_pool *pool) class->size, fg); } } + kfree(class); } kfree(pool); } @@ -999,7 +1063,6 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size) { unsigned long obj; struct link_free *link; - int class_idx; struct size_class *class; struct page *first_page, *m_page; @@ -1008,9 +1071,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size) if (unlikely(!size || size > ZS_MAX_ALLOC_SIZE)) return 0; - class_idx = get_size_class_index(size); - class = &pool->size_class[class_idx]; - BUG_ON(class_idx != class->index); + class = pool->size_class[get_size_class_index(size)]; spin_lock(&class->lock); first_page = find_get_zspage(class); @@ -1063,7 +1124,7 @@ void zs_free(struct zs_pool *pool, unsigned long obj) first_page = get_first_page(f_page); get_zspage_mapping(first_page, &class_idx, &fullness); - class = &pool->size_class[class_idx]; + class = pool->size_class[class_idx]; f_offset = obj_idx_to_offset(f_page, f_objidx, class->size); spin_lock(&class->lock); @@ -1124,7 +1185,7 @@ void *zs_map_object(struct zs_pool *pool, unsigned long handle, obj_handle_to_location(handle, &page, &obj_idx); get_zspage_mapping(get_first_page(page), &class_idx, &fg); - class = &pool->size_class[class_idx]; + class = pool->size_class[class_idx]; off = obj_idx_to_offset(page, obj_idx, class->size); area = &get_cpu_var(zs_map_area); @@ -1158,7 +1219,7 @@ void zs_unmap_object(struct zs_pool *pool, unsigned long handle) obj_handle_to_location(handle, &page, &obj_idx); get_zspage_mapping(get_first_page(page), &class_idx, &fg); - class = &pool->size_class[class_idx]; + class = pool->size_class[class_idx]; off = obj_idx_to_offset(page, obj_idx, class->size); area = this_cpu_ptr(&zs_map_area);