Centos 7 systemd 死锁问题分析

struct kmem_cache *
kmem_cache_create_memcg(struct mem_cgroup *memcg, const char *name, size_t size,
			size_t align, unsigned long flags, void (*ctor)(void *),
			struct kmem_cache *parent_cache)
{
	printk(KERN_DEBUG
			"kdbg: creating cache %s with cgroup %p and parent cache %p",
			name, memcg, parent_cache);
	struct kmem_cache *s = NULL;
	int err = 0;

	get_online_cpus();
	mutex_lock(&slab_mutex);

	if (!kmem_cache_sanity_check(memcg, name, size) == 0)
		goto out_locked;

	/*
	 * Some allocators will constraint the set of valid flags to a subset
	 * of all flags. We expect them to define CACHE_CREATE_MASK in this
	 * case, and we'll just provide them with a sanitized version of the
	 * passed flags.
	 */
	flags &= CACHE_CREATE_MASK;

	s = __kmem_cache_alias(memcg, name, size, align, flags, ctor);
	if (s) {
		printk(KERN_DEBUG "kdbg: cache %s(%p) shares %s with refcount %d and %s",
				name, s, s->name, s->refcount, is_root_cache(s)?"root":"nonroot");
		goto out_locked;
	}

	s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
	if (s) {
		s->object_size = s->size = size;
		s->align = calculate_alignment(flags, align, size);
		s->ctor = ctor;

		if (memcg_register_cache(memcg, s, parent_cache)) {
			kmem_cache_free(kmem_cache, s);
			err = -ENOMEM;
			goto out_locked;
		}

		s->name = kstrdup(name, GFP_KERNEL);
		if (!s->name) {
			kmem_cache_free(kmem_cache, s);
			err = -ENOMEM;
			goto out_locked;
		}

		err = __kmem_cache_create(s, flags);
		if (!err) {
			s->refcount = 1;
			list_add(&s->list, &slab_caches);
			memcg_cache_list_add(memcg, s);
			printk(KERN_DEBUG "kdbg: allocated %s(%p) with refcount %d and %s",
					name, s, s->refcount, is_root_cache(s)?"root":"nonroot");
		} else {
			kfree(s->name);
			kmem_cache_free(kmem_cache, s);
		}
	} else
		err = -ENOMEM;

out_locked:
	mutex_unlock(&slab_mutex);
	put_online_cpus();

	if (err) {

		if (flags & SLAB_PANIC)
			panic("kmem_cache_create: Failed to create slab'%s'. Error %d\n",
				name, err);
		else {
			printk(KERN_WARNING "kmem_cache_create(%s) failed with error %d",
				name, err);
			dump_stack();
		}

		return NULL;
	}

	return s;
}

struct kmem_cache *
kmem_cache_create(const char *name, size_t size, size_t align,
		  unsigned long flags, void (*ctor)(void *))
{
	return kmem_cache_create_memcg(NULL, name, size, align, flags, ctor, NULL);
}
EXPORT_SYMBOL(kmem_cache_create);

void kmem_cache_destroy(struct kmem_cache *s)
{
	if (unlikely(!s))
		return;

	printk(KERN_DEBUG "kdbg: recycle cache %p which is %s", s,
			is_root_cache(s)?"root":"nonroot");

	/* Destroy all the children caches if we aren't a memcg cache */
	kmem_cache_destroy_memcg_children(s);

	get_online_cpus();
	mutex_lock(&slab_mutex);
	s->refcount--;
	if (!s->refcount) {
		list_del(&s->list);

		if (!__kmem_cache_shutdown(s)) {
			mutex_unlock(&slab_mutex);
			if (s->flags & SLAB_DESTROY_BY_RCU)
				rcu_barrier();

			memcg_release_cache(s);
			kfree(s->name);
			kmem_cache_free(kmem_cache, s);
			printk(KERN_DEBUG "kdbg: recycled cache %p", s);
		} else {
			list_add(&s->list, &slab_caches);
			mutex_unlock(&slab_mutex);
			printk(KERN_ERR "kmem_cache_destroy %s: Slab cache still has objects\n",
				s->name);
			dump_stack();
		}
	} else {
		mutex_unlock(&slab_mutex);
	}
	put_online_cpus();
}

void memcg_release_cache(struct kmem_cache *s)
{
	struct kmem_cache *root;
	struct mem_cgroup *memcg;
	int id;

	/*
	 * This happens, for instance, when a root cache goes away before we
	 * add any memcg.
	 */
	if (!s->memcg_params)
		return;

	if (s->memcg_params->is_root_cache)
		goto out;

	memcg = s->memcg_params->memcg;
	id  = memcg_cache_id(memcg);

	root = s->memcg_params->root_cache;
	root->memcg_params->memcg_caches[id] = NULL;
	printk(KERN_DEBUG "kdbg: remove cache %p from parent %p with index %d",
			s, root, id);

	mutex_lock(&memcg->slab_caches_mutex);
	list_del(&s->memcg_params->list);
	mutex_unlock(&memcg->slab_caches_mutex);

	mem_cgroup_put(memcg);
out:
	kfree(s->memcg_params);
}

static void kmem_cache_destroy_work_func(struct work_struct *w)
{
	struct kmem_cache *cachep;
	struct memcg_cache_params *p;

	p = container_of(w, struct memcg_cache_params, destroy);

	cachep = memcg_params_to_cache(p);

	printk(KERN_DEBUG "kdbg: destroy worker for %p is called", cachep);

	/*
	 * If we get down to 0 after shrink, we could delete right away.
	 * However, memcg_release_pages() already puts us back in the workqueue
	 * in that case. If we proceed deleting, we'll get a dangling
	 * reference, and removing the object from the workqueue in that case
	 * is unnecessary complication. We are not a fast path.
	 *
	 * Note that this case is fundamentally different from racing with
	 * shrink_slab(): if memcg_cgroup_destroy_cache() is called in
	 * kmem_cache_shrink, not only we would be reinserting a dead cache
	 * into the queue, but doing so from inside the worker racing to
	 * destroy it.
	 *
	 * So if we aren't down to zero, we'll just schedule a worker and try
	 * again
	 */
	if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
		kmem_cache_shrink(cachep);
		if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
			return;
	} else {
		printk(KERN_DEBUG "kdbg: destroy %p", cachep);
		kmem_cache_destroy(cachep);
	}
}

void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
{
	struct kmem_cache *c;
	int i;
	char task_comm[TASK_COMM_LEN + 1];
	task_comm[TASK_COMM_LEN] = 0;

	if (!s->memcg_params)
		return;
	if (!s->memcg_params->is_root_cache)
		return;

	if (memcg_limit_mutex.owner == current) {
		/* Get task command name. */
		get_task_comm(task_comm, memcg_limit_mutex.owner);
		printk(KERN_EMERG "kdbg: %s dead lock with cache %p", task_comm, s);
	}

	/*
	 * If the cache is being destroyed, we trust that there is no one else
	 * requesting objects from it. Even if there are, the sanity checks in
	 * kmem_cache_destroy should caught this ill-case.
	 *
	 * Still, we don't want anyone else freeing memcg_caches under our
	 * noses, which can happen if a new memcg comes to life. As usual,
	 * we'll take the memcg_limit_mutex to protect ourselves against this.
	 */
	mutex_lock(&memcg_limit_mutex);
	for (i = 0; i < memcg_limited_groups_array_size; i++) {
		c = s->memcg_params->memcg_caches[i];
		if (!c)
			continue;

		/*
		 * We will now manually delete the caches, so to avoid races
		 * we need to cancel all pending destruction workers and
		 * proceed with destruction ourselves.
		 *
		 * kmem_cache_destroy() will call kmem_cache_shrink internally,
		 * and that could spawn the workers again: it is likely that
		 * the cache still have active pages until this very moment.
		 * This would lead us back to mem_cgroup_destroy_cache.
		 *
		 * But that will not execute at all if the "dead" flag is not
		 * set, so flip it down to guarantee we are in control.
		 */
		c->memcg_params->dead = false;
		cancel_work_sync(&c->memcg_params->destroy);

		printk(KERN_DEBUG "kdbg: parent %p cache %p index %d is %s existing %p",
				s, c, i, is_root_cache(s)?"root":"nonroot",
				s->memcg_params->memcg_caches[i]);
		kmem_cache_destroy(c);
	}
	mutex_unlock(&memcg_limit_mutex);
}

	mutex_lock(&memcg_limit_mutex);
	for (i = 0; i < memcg_limited_groups_array_size; i++) {
		c = s->memcg_params->memcg_caches[i];
		if (!c)
			continue;
		c->memcg_params->dead = false;
		cancel_work_sync(&c->memcg_params->destroy);
		/*
		 * 问题就在这里，此时 s->memcg_params->memcg_caches[i] 已经为 NULL 了,
		 * 也就是说，不应该再对 c 进行清理操作。
		 */
		printk(KERN_DEBUG "kdbg: parent %p cache %p index %d is %s existing %p",
				s, c, i, is_root_cache(s)?"root":"nonroot",
				s->memcg_params->memcg_caches[i]);
		kmem_cache_destroy(c);
	}
	mutex_unlock(&memcg_limit_mutex);