drivers/gpu/drm/drm_sync_helper.c - chromiumos/third_party/kernel - Git at Google

 /*
  * drm_sync_helper.c: software fence and helper functions for fences and
  * reservations used for dma buffer access synchronization between drivers.
  *
  * Copyright 2014 Google, Inc.
  *
  * This software is licensed under the terms of the GNU General Public
  * License version 2, as published by the Free Software Foundation, and
  * may be copied, distributed, and modified under those terms.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  */

 #include <linux/module.h>
 #include <drm/drm_sync_helper.h>
 #include <linux/slab.h>
 #include <linux/reservation.h>

 static DEFINE_SPINLOCK(sw_fence_lock);

 void drm_add_reservation(struct reservation_object *resv,
 			 struct reservation_object **resvs,
 			 unsigned long *excl_resvs_bitmap,
 			 unsigned int *num_resvs, bool exclusive)
 {
 	unsigned int r;

 	for (r = 0; r < *num_resvs; r++) {
 		if (resvs[r] == resv)
 			return;
 	}
 	resvs[*num_resvs] = resv;
 	if (exclusive)
 		set_bit(*num_resvs, excl_resvs_bitmap);
 	(*num_resvs)++;
 }
 EXPORT_SYMBOL(drm_add_reservation);

 int drm_lock_reservations(struct reservation_object **resvs,
 			  unsigned int num_resvs, struct ww_acquire_ctx *ctx)
 {
 	unsigned int r;
 	struct reservation_object *slow_res = NULL;

 	ww_acquire_init(ctx, &reservation_ww_class);

 retry:
 	for (r = 0; r < num_resvs; r++) {
 		int ret;
 		/* skip the resv we locked with slow lock */
 		if (resvs[r] == slow_res) {
 			slow_res = NULL;
 			continue;
 		}
 		ret = ww_mutex_lock(&resvs[r]->lock, ctx);
 		if (ret < 0) {
 			unsigned int slow_r = r;
 			/*
 			 * undo all the locks we already done,
 			 * in reverse order
 			 */
 			while (r > 0) {
 				r--;
 				ww_mutex_unlock(&resvs[r]->lock);
 			}
 			if (slow_res)
 				ww_mutex_unlock(&slow_res->lock);
 			if (ret == -EDEADLK) {
 				slow_res = resvs[slow_r];
 				ww_mutex_lock_slow(&slow_res->lock, ctx);
 				goto retry;
 			}
 			ww_acquire_fini(ctx);
 			return ret;
 		}
 	}

 	ww_acquire_done(ctx);
 	return 0;
 }
 EXPORT_SYMBOL(drm_lock_reservations);

 void drm_unlock_reservations(struct reservation_object **resvs,
 			     unsigned int num_resvs,
 			     struct ww_acquire_ctx *ctx)
 {
 	unsigned int r;

 	for (r = 0; r < num_resvs; r++)
 		ww_mutex_unlock(&resvs[r]->lock);

 	ww_acquire_fini(ctx);
 }
 EXPORT_SYMBOL(drm_unlock_reservations);

 static void reservation_cb_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 {
 	struct drm_reservation_fence_cb *rfcb =
 		container_of(cb, struct drm_reservation_fence_cb, base);
 	struct drm_reservation_cb *rcb = rfcb->parent;

 	if (atomic_dec_and_test(&rcb->count))
 		schedule_work(&rcb->work);
 }

 static void
 reservation_cb_cleanup(struct drm_reservation_cb *rcb)
 {
 	unsigned cb;

 	for (cb = 0; cb < rcb->num_fence_cbs; cb++) {
 		if (rcb->fence_cbs[cb]) {
 			dma_fence_remove_callback(rcb->fence_cbs[cb]->fence,
 						  &rcb->fence_cbs[cb]->base);
 			dma_fence_put(rcb->fence_cbs[cb]->fence);
 			kfree(rcb->fence_cbs[cb]);
 			rcb->fence_cbs[cb] = NULL;
 		}
 	}
 	kfree(rcb->fence_cbs);
 	rcb->fence_cbs = NULL;
 	rcb->num_fence_cbs = 0;
 }

 static void reservation_cb_work(struct work_struct *pwork)
 {
 	struct drm_reservation_cb *rcb =
 		container_of(pwork, struct drm_reservation_cb, work);
 	/*
 	 * clean up everything before calling the callback, because the callback
 	 * may free structure containing rcb and work_struct
 	 */
 	reservation_cb_cleanup(rcb);
 	rcb->func(rcb, rcb->context);
 }

 static int
 reservation_cb_add_fence_cb(struct drm_reservation_cb *rcb, struct dma_fence *fence)
 {
 	int ret = 0;
 	struct drm_reservation_fence_cb *fence_cb;
 	struct drm_reservation_fence_cb **new_fence_cbs;

 	new_fence_cbs = krealloc(rcb->fence_cbs,
 				(rcb->num_fence_cbs + 1)
 				* sizeof(struct drm_reservation_fence_cb *),
 				GFP_KERNEL);
 	if (!new_fence_cbs)
 		return -ENOMEM;
 	rcb->fence_cbs = new_fence_cbs;

 	fence_cb = kzalloc(sizeof(struct drm_reservation_fence_cb), GFP_KERNEL);
 	if (!fence_cb)
 		return -ENOMEM;

 	/*
 	 * do not want for fence to disappear on us while we are waiting for
 	 * callback and we need it in case we want to remove callbacks
 	 */
 	dma_fence_get(fence);
 	fence_cb->fence = fence;
 	fence_cb->parent = rcb;
 	rcb->fence_cbs[rcb->num_fence_cbs] = fence_cb;
 	atomic_inc(&rcb->count);
 	ret = dma_fence_add_callback(fence, &fence_cb->base,
 					reservation_cb_fence_cb);
 	if (ret == -ENOENT) {
 		/* already signaled */
 		atomic_dec(&rcb->count);
 		dma_fence_put(fence_cb->fence);
 		kfree(fence_cb);
 		ret = 0;
 	} else if (ret < 0) {
 		atomic_dec(&rcb->count);
 		dma_fence_put(fence_cb->fence);
 		kfree(fence_cb);
 		return ret;
 	} else {
 		rcb->num_fence_cbs++;
 	}
 	return ret;
 }

 void
 drm_reservation_cb_init(struct drm_reservation_cb *rcb,
 			drm_reservation_cb_func_t func, void *context)
 {
 	INIT_WORK(&rcb->work, reservation_cb_work);
 	atomic_set(&rcb->count, 1);
 	rcb->num_fence_cbs = 0;
 	rcb->fence_cbs = NULL;
 	rcb->func = func;
 	rcb->context = context;
 }
 EXPORT_SYMBOL(drm_reservation_cb_init);

 int
 drm_reservation_cb_add(struct drm_reservation_cb *rcb,
 			struct reservation_object *resv, bool exclusive)
 {
 	int ret = 0;
 	struct dma_fence *fence;
 	unsigned shared_count = 0, f;
 	struct dma_fence **shared_fences = NULL;

 	/* enum all the fences in the reservation and add callbacks */
 	ret = reservation_object_get_fences_rcu(resv, &fence,
 					&shared_count, &shared_fences);
 	if (ret < 0)
 		return ret;

 	if (fence) {
 		ret = reservation_cb_add_fence_cb(rcb, fence);
 		if (ret < 0) {
 			reservation_cb_cleanup(rcb);
 			goto error;
 		}
 	}

 	if (exclusive) {
 		for (f = 0; f < shared_count; f++) {
 			ret = reservation_cb_add_fence_cb(rcb,
 						shared_fences[f]);
 			if (ret < 0) {
 				reservation_cb_cleanup(rcb);
 				goto error;
 			}
 		}
 	}

 error:
 	if (fence)
 		dma_fence_put(fence);
 	if (shared_fences) {
 		for (f = 0; f < shared_count; f++)
 			dma_fence_put(shared_fences[f]);
 		kfree(shared_fences);
 	}
 	return ret;
 }
 EXPORT_SYMBOL(drm_reservation_cb_add);

 void
 drm_reservation_cb_done(struct drm_reservation_cb *rcb)
 {
 	/*
 	 * we need to decrement from initial 1
 	 * and trigger the callback in case all the
 	 * fences were already triggered
 	 */
 	if (atomic_dec_and_test(&rcb->count)) {
 		/*
 		 * we could call the callback here directly but in case
 		 * the callback function needs to lock the same mutex
 		 * as our caller it could cause a deadlock, so it is
 		 * safer to call it from a worker
 		 */
 		schedule_work(&rcb->work);
 	}
 }
 EXPORT_SYMBOL(drm_reservation_cb_done);

 void
 drm_reservation_cb_fini(struct drm_reservation_cb *rcb)
 {
 	/* make sure no work will be triggered */
 	atomic_set(&rcb->count, 0);
 	cancel_work_sync(&rcb->work);
 	reservation_cb_cleanup(rcb);
 }
 EXPORT_SYMBOL(drm_reservation_cb_fini);

 static bool sw_fence_enable_signaling(struct dma_fence *f)
 {
 	return true;
 }

 static const char *sw_fence_get_get_driver_name(struct dma_fence *fence)
 {
 	return "drm_sync_helper";
 }

 static const char *sw_fence_get_timeline_name(struct dma_fence *f)
 {
 	return "drm_sync.sw";
 }

 static const struct dma_fence_ops sw_fence_ops = {
 	.get_driver_name = sw_fence_get_get_driver_name,
 	.get_timeline_name = sw_fence_get_timeline_name,
 	.enable_signaling = sw_fence_enable_signaling,
 	.signaled = NULL,
 	.wait = dma_fence_default_wait,
 	.release = NULL
 };

 struct dma_fence *drm_sw_fence_new(unsigned int context, unsigned seqno)
 {
 	struct dma_fence *fence;

 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 	if (!fence)
 		return ERR_PTR(-ENOMEM);
 	dma_fence_init(fence,
 		       &sw_fence_ops,
 		       &sw_fence_lock,
 		       context, seqno);

 	return fence;
 }
 EXPORT_SYMBOL(drm_sw_fence_new);
	/*
	* drm_sync_helper.c: software fence and helper functions for fences and
	* reservations used for dma buffer access synchronization between drivers.
	*
	* Copyright 2014 Google, Inc.
	*
	* This software is licensed under the terms of the GNU General Public
	* License version 2, as published by the Free Software Foundation, and
	* may be copied, distributed, and modified under those terms.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*/

	#include <linux/module.h>
	#include <drm/drm_sync_helper.h>
	#include <linux/slab.h>
	#include <linux/reservation.h>

	static DEFINE_SPINLOCK(sw_fence_lock);

	void drm_add_reservation(struct reservation_object *resv,
	struct reservation_object **resvs,
	unsigned long *excl_resvs_bitmap,
	unsigned int *num_resvs, bool exclusive)
	{
	unsigned int r;

	for (r = 0; r < *num_resvs; r++) {
	if (resvs[r] == resv)
	return;
	}
	resvs[*num_resvs] = resv;
	if (exclusive)
	set_bit(*num_resvs, excl_resvs_bitmap);
	(*num_resvs)++;
	}
	EXPORT_SYMBOL(drm_add_reservation);

	int drm_lock_reservations(struct reservation_object **resvs,
	unsigned int num_resvs, struct ww_acquire_ctx *ctx)
	{
	unsigned int r;
	struct reservation_object *slow_res = NULL;

	ww_acquire_init(ctx, &reservation_ww_class);

	retry:
	for (r = 0; r < num_resvs; r++) {
	int ret;
	/* skip the resv we locked with slow lock */
	if (resvs[r] == slow_res) {
	slow_res = NULL;
	continue;
	}
	ret = ww_mutex_lock(&resvs[r]->lock, ctx);
	if (ret < 0) {
	unsigned int slow_r = r;
	/*
	* undo all the locks we already done,
	* in reverse order
	*/
	while (r > 0) {
	r--;
	ww_mutex_unlock(&resvs[r]->lock);
	}
	if (slow_res)
	ww_mutex_unlock(&slow_res->lock);
	if (ret == -EDEADLK) {
	slow_res = resvs[slow_r];
	ww_mutex_lock_slow(&slow_res->lock, ctx);
	goto retry;
	}
	ww_acquire_fini(ctx);
	return ret;
	}
	}

	ww_acquire_done(ctx);
	return 0;
	}
	EXPORT_SYMBOL(drm_lock_reservations);

	void drm_unlock_reservations(struct reservation_object **resvs,
	unsigned int num_resvs,
	struct ww_acquire_ctx *ctx)
	{
	unsigned int r;

	for (r = 0; r < num_resvs; r++)
	ww_mutex_unlock(&resvs[r]->lock);

	ww_acquire_fini(ctx);
	}
	EXPORT_SYMBOL(drm_unlock_reservations);

	static void reservation_cb_fence_cb(struct dma_fence fence, struct dma_fence_cb cb)
	{
	struct drm_reservation_fence_cb *rfcb =
	container_of(cb, struct drm_reservation_fence_cb, base);
	struct drm_reservation_cb *rcb = rfcb->parent;

	if (atomic_dec_and_test(&rcb->count))
	schedule_work(&rcb->work);
	}

	static void
	reservation_cb_cleanup(struct drm_reservation_cb *rcb)
	{
	unsigned cb;

	for (cb = 0; cb < rcb->num_fence_cbs; cb++) {
	if (rcb->fence_cbs[cb]) {
	dma_fence_remove_callback(rcb->fence_cbs[cb]->fence,
	&rcb->fence_cbs[cb]->base);
	dma_fence_put(rcb->fence_cbs[cb]->fence);
	kfree(rcb->fence_cbs[cb]);
	rcb->fence_cbs[cb] = NULL;
	}
	}
	kfree(rcb->fence_cbs);
	rcb->fence_cbs = NULL;
	rcb->num_fence_cbs = 0;
	}

	static void reservation_cb_work(struct work_struct *pwork)
	{
	struct drm_reservation_cb *rcb =
	container_of(pwork, struct drm_reservation_cb, work);
	/*
	* clean up everything before calling the callback, because the callback
	* may free structure containing rcb and work_struct
	*/
	reservation_cb_cleanup(rcb);
	rcb->func(rcb, rcb->context);
	}

	static int
	reservation_cb_add_fence_cb(struct drm_reservation_cb rcb, struct dma_fence fence)
	{
	int ret = 0;
	struct drm_reservation_fence_cb *fence_cb;
	struct drm_reservation_fence_cb **new_fence_cbs;

	new_fence_cbs = krealloc(rcb->fence_cbs,
	(rcb->num_fence_cbs + 1)
	* sizeof(struct drm_reservation_fence_cb *),
	GFP_KERNEL);
	if (!new_fence_cbs)
	return -ENOMEM;
	rcb->fence_cbs = new_fence_cbs;

	fence_cb = kzalloc(sizeof(struct drm_reservation_fence_cb), GFP_KERNEL);
	if (!fence_cb)
	return -ENOMEM;

	/*
	* do not want for fence to disappear on us while we are waiting for
	* callback and we need it in case we want to remove callbacks
	*/
	dma_fence_get(fence);
	fence_cb->fence = fence;
	fence_cb->parent = rcb;
	rcb->fence_cbs[rcb->num_fence_cbs] = fence_cb;
	atomic_inc(&rcb->count);
	ret = dma_fence_add_callback(fence, &fence_cb->base,
	reservation_cb_fence_cb);
	if (ret == -ENOENT) {
	/* already signaled */
	atomic_dec(&rcb->count);
	dma_fence_put(fence_cb->fence);
	kfree(fence_cb);
	ret = 0;
	} else if (ret < 0) {
	atomic_dec(&rcb->count);
	dma_fence_put(fence_cb->fence);
	kfree(fence_cb);
	return ret;
	} else {
	rcb->num_fence_cbs++;
	}
	return ret;
	}

	void
	drm_reservation_cb_init(struct drm_reservation_cb *rcb,
	drm_reservation_cb_func_t func, void *context)
	{
	INIT_WORK(&rcb->work, reservation_cb_work);
	atomic_set(&rcb->count, 1);
	rcb->num_fence_cbs = 0;
	rcb->fence_cbs = NULL;
	rcb->func = func;
	rcb->context = context;
	}
	EXPORT_SYMBOL(drm_reservation_cb_init);

	int
	drm_reservation_cb_add(struct drm_reservation_cb *rcb,
	struct reservation_object *resv, bool exclusive)
	{
	int ret = 0;
	struct dma_fence *fence;
	unsigned shared_count = 0, f;
	struct dma_fence **shared_fences = NULL;

	/* enum all the fences in the reservation and add callbacks */
	ret = reservation_object_get_fences_rcu(resv, &fence,
	&shared_count, &shared_fences);
	if (ret < 0)
	return ret;

	if (fence) {
	ret = reservation_cb_add_fence_cb(rcb, fence);
	if (ret < 0) {
	reservation_cb_cleanup(rcb);
	goto error;
	}
	}

	if (exclusive) {
	for (f = 0; f < shared_count; f++) {
	ret = reservation_cb_add_fence_cb(rcb,
	shared_fences[f]);
	if (ret < 0) {
	reservation_cb_cleanup(rcb);
	goto error;
	}
	}
	}

	error:
	if (fence)
	dma_fence_put(fence);
	if (shared_fences) {
	for (f = 0; f < shared_count; f++)
	dma_fence_put(shared_fences[f]);
	kfree(shared_fences);
	}
	return ret;
	}
	EXPORT_SYMBOL(drm_reservation_cb_add);

	void
	drm_reservation_cb_done(struct drm_reservation_cb *rcb)
	{
	/*
	* we need to decrement from initial 1
	* and trigger the callback in case all the
	* fences were already triggered
	*/
	if (atomic_dec_and_test(&rcb->count)) {
	/*
	* we could call the callback here directly but in case
	* the callback function needs to lock the same mutex
	* as our caller it could cause a deadlock, so it is
	* safer to call it from a worker
	*/
	schedule_work(&rcb->work);
	}
	}
	EXPORT_SYMBOL(drm_reservation_cb_done);

	void
	drm_reservation_cb_fini(struct drm_reservation_cb *rcb)
	{
	/* make sure no work will be triggered */
	atomic_set(&rcb->count, 0);
	cancel_work_sync(&rcb->work);
	reservation_cb_cleanup(rcb);
	}
	EXPORT_SYMBOL(drm_reservation_cb_fini);

	static bool sw_fence_enable_signaling(struct dma_fence *f)
	{
	return true;
	}

	static const char sw_fence_get_get_driver_name(struct dma_fence fence)
	{
	return "drm_sync_helper";
	}

	static const char sw_fence_get_timeline_name(struct dma_fence f)
	{
	return "drm_sync.sw";
	}

	static const struct dma_fence_ops sw_fence_ops = {
	.get_driver_name = sw_fence_get_get_driver_name,
	.get_timeline_name = sw_fence_get_timeline_name,
	.enable_signaling = sw_fence_enable_signaling,
	.signaled = NULL,
	.wait = dma_fence_default_wait,
	.release = NULL
	};

	struct dma_fence *drm_sw_fence_new(unsigned int context, unsigned seqno)
	{
	struct dma_fence *fence;

	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	if (!fence)
	return ERR_PTR(-ENOMEM);
	dma_fence_init(fence,
	&sw_fence_ops,
	&sw_fence_lock,
	context, seqno);

	return fence;
	}
	EXPORT_SYMBOL(drm_sw_fence_new);