/*
 *
 * (C) COPYRIGHT 2010-2013 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU licence.
 *
 * A copy of the licence is included with the program, and can also be obtained
 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA  02110-1301, USA.
 *
 */



#if defined(CONFIG_DMA_SHARED_BUFFER)
#include <linux/dma-buf.h>
#endif				/* defined(CONFIG_DMA_SHARED_BUFFER) */
#ifdef CONFIG_COMPAT
#include <linux/compat.h>
#endif
#include <kbase/src/common/mali_kbase.h>
#include <kbase/src/common/mali_kbase_uku.h>
#include <kbase/src/common/mali_kbase_js_affinity.h>
#include <kbase/src/common/mali_kbase_10969_workaround.h>
#ifdef CONFIG_UMP
#include <linux/ump.h>
#endif				/* CONFIG_UMP */

#define beenthere(f, a...)  KBASE_DEBUG_PRINT_INFO(KBASE_JD, "%s:" f, __func__, ##a)

/*
 * This is the kernel side of the API. Only entry points are:
 * - kbase_jd_submit(): Called from userspace to submit a single bag
 * - kbase_jd_done(): Called from interrupt context to track the
 *   completion of a job.
 * Callouts:
 * - to the job manager (enqueue a job)
 * - to the event subsystem (signals the completion/failure of bag/job-chains).
 */

static void *get_compat_pointer(const kbase_pointer *p)
{
#ifdef CONFIG_COMPAT
	if (is_compat_task())
		return (void *)p->compat_value;
	else
#endif
		return p->value;
}

/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs
 *
 * Returns whether the JS needs a reschedule.
 *
 * Note that the caller must also check the atom status and
 * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock
 */
static int jd_run_atom(kbase_jd_atom *katom)
{
	kbase_context *kctx = katom->kctx;
	KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED);

	if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) {
		/* Dependency only atom */
		katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
		return 0;
	} else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) {
		/* Soft-job */
		if (kbase_process_soft_job(katom) == 0) {
			kbase_finish_soft_job(katom);
			katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
		} else {
			/* The job has not completed */
			list_add_tail(&katom->dep_item[0], &kctx->waiting_soft_jobs);
		}
		return 0;
	}
	katom->status = KBASE_JD_ATOM_STATE_IN_JS;
	/* Queue an action about whether we should try scheduling a context */
	return kbasep_js_add_job(kctx, katom);
}

#ifdef CONFIG_KDS

/* Add the katom to the kds waiting list.
 * Atoms must be added to the waiting list after a successful call to kds_async_waitall.
 * The caller must hold the kbase_jd_context.lock */

static void kbase_jd_kds_waiters_add(kbase_jd_atom *katom)
{
	kbase_context *kctx;
	KBASE_DEBUG_ASSERT(katom);

	kctx = katom->kctx;

	list_add_tail(&katom->node, &kctx->waiting_kds_resource);
}

/* Remove the katom from the kds waiting list.
 * Atoms must be removed from the waiting list before a call to kds_resource_set_release_sync.
 * The supplied katom must first have been added to the list with a call to kbase_jd_kds_waiters_add.
 * The caller must hold the kbase_jd_context.lock */

static void kbase_jd_kds_waiters_remove(kbase_jd_atom *katom)
{
	KBASE_DEBUG_ASSERT(katom);
	list_del(&katom->node);
}

static void kds_dep_clear(void *callback_parameter, void *callback_extra_parameter)
{
	kbase_jd_atom *katom;
	kbase_jd_context *ctx;
	kbase_device *kbdev;

	katom = (kbase_jd_atom *) callback_parameter;
	KBASE_DEBUG_ASSERT(katom);
	ctx = &katom->kctx->jctx;
	kbdev = katom->kctx->kbdev;
	KBASE_DEBUG_ASSERT(kbdev);

	mutex_lock(&ctx->lock);

	/* KDS resource has already been satisfied (e.g. due to zapping) */
	if (katom->kds_dep_satisfied)
		goto out;

	/* This atom's KDS dependency has now been met */
	katom->kds_dep_satisfied = MALI_TRUE;

	/* Check whether the atom's other dependencies were already met */
	if (!katom->dep_atom[0] && !katom->dep_atom[1]) {
		/* katom dep complete, attempt to run it */
		mali_bool resched = MALI_FALSE;
		resched = jd_run_atom(katom);

		if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
			/* The atom has already finished */
			resched |= jd_done_nolock(katom);
		}

		if (resched)
			kbasep_js_try_schedule_head_ctx(kbdev);
	}
 out:
	mutex_unlock(&ctx->lock);
}

void kbase_cancel_kds_wait_job(kbase_jd_atom *katom)
{
	KBASE_DEBUG_ASSERT(katom);

	/* Prevent job_done_nolock from being called twice on an atom when
	 *  there is a race between job completion and cancellation */

	if ( katom->status == KBASE_JD_ATOM_STATE_QUEUED ) {
		/* Wait was cancelled - zap the atom */
		katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
		if (jd_done_nolock(katom)) {
			kbasep_js_try_schedule_head_ctx( katom->kctx->kbdev );
		}
	}
}
#endif				/* CONFIG_KDS */

#ifdef CONFIG_DMA_SHARED_BUFFER
static mali_error kbase_jd_umm_map(kbase_context *kctx, struct kbase_va_region *reg)
{
	struct sg_table *st;
	struct scatterlist *s;
	int i;
	phys_addr_t *pa;
	mali_error err;
	size_t count = 0;

	KBASE_DEBUG_ASSERT(NULL == reg->imported_metadata.umm.st);
	st = dma_buf_map_attachment(reg->imported_metadata.umm.dma_attachment, DMA_BIDIRECTIONAL);

	if (IS_ERR_OR_NULL(st))
		return MALI_ERROR_FUNCTION_FAILED;

	/* save for later */
	reg->imported_metadata.umm.st = st;

	pa = kbase_get_phy_pages(reg);
	KBASE_DEBUG_ASSERT(pa);

	for_each_sg(st->sgl, s, st->nents, i) {
		int j;
		size_t pages = PFN_DOWN(sg_dma_len(s));

		for (j = 0; (j < pages) && (count < reg->nr_pages); j++, count++)
			*pa++ = sg_dma_address(s) + (j << PAGE_SHIFT);
		WARN_ONCE(j < pages, "sg list returned by dma_buf_map_attachment is larger than dma_buf->size=%zu\n", reg->imported_metadata.umm.dma_buf->size);
	}

	err = kbase_mmu_insert_pages(kctx, reg->start_pfn, kbase_get_phy_pages(reg), reg->nr_alloc_pages, reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD);

	if (MALI_ERROR_NONE != err) {
		dma_buf_unmap_attachment(reg->imported_metadata.umm.dma_attachment, reg->imported_metadata.umm.st, DMA_BIDIRECTIONAL);
		reg->imported_metadata.umm.st = NULL;
	}

	return err;
}

static void kbase_jd_umm_unmap(kbase_context *kctx, struct kbase_va_region *reg)
{
	KBASE_DEBUG_ASSERT(kctx);
	KBASE_DEBUG_ASSERT(reg);
	KBASE_DEBUG_ASSERT(reg->imported_metadata.umm.dma_attachment);
	KBASE_DEBUG_ASSERT(reg->imported_metadata.umm.st);
	kbase_mmu_teardown_pages(kctx, reg->start_pfn, reg->nr_alloc_pages);
	dma_buf_unmap_attachment(reg->imported_metadata.umm.dma_attachment, reg->imported_metadata.umm.st, DMA_BIDIRECTIONAL);
	reg->imported_metadata.umm.st = NULL;
}
#endif				/* CONFIG_DMA_SHARED_BUFFER */

void kbase_jd_free_external_resources(kbase_jd_atom *katom)
{
#ifdef CONFIG_KDS
	if (katom->kds_rset) {
		kbase_jd_context * jctx = &katom->kctx->jctx;

		/*
		 * As the atom is no longer waiting, remove it from
		 * the waiting list.
		 */

		mutex_lock(&jctx->lock);
		kbase_jd_kds_waiters_remove( katom );
		mutex_unlock(&jctx->lock);

		/* Release the kds resource or cancel if zapping */
		kds_resource_set_release_sync(&katom->kds_rset);
	}
#endif				/* CONFIG_KDS */
}

static void kbase_jd_post_external_resources(kbase_jd_atom *katom)
{
	KBASE_DEBUG_ASSERT(katom);
	KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES);

#ifdef CONFIG_KDS
	/* Prevent the KDS resource from triggering the atom in case of zapping */
	if (katom->kds_rset)
		katom->kds_dep_satisfied = MALI_TRUE;
#endif				/* CONFIG_KDS */

#if defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG)
	/* Lock also used in debug mode just for lock order checking */
	kbase_gpu_vm_lock(katom->kctx);
#endif				/* defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG) */
	/* only roll back if extres is non-NULL */
	if (katom->extres) {
#ifdef CONFIG_DMA_SHARED_BUFFER
		u32 res_no;
		res_no = katom->nr_extres;
		while (res_no-- > 0) {
			base_external_resource *res;
			kbase_va_region *reg;

			res = &katom->extres[res_no];
			reg = kbase_region_tracker_find_region_enclosing_address(katom->kctx, res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE);
			/* if reg wasn't found then it has been freed while the job ran */
			if (reg && reg->imported_type == BASE_TMEM_IMPORT_TYPE_UMM) {
				/* last job using */
				if (1 == reg->imported_metadata.umm.current_mapping_usage_count--)
					kbase_jd_umm_unmap(katom->kctx, reg);
			}
		}
#endif				/* CONFIG_DMA_SHARED_BUFFER */
		kfree(katom->extres);
		katom->extres = NULL;
	}
#if defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG)
	/* Lock also used in debug mode just for lock order checking */
	kbase_gpu_vm_unlock(katom->kctx);
#endif				/* defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG) */
}

#if (defined(CONFIG_KDS) && defined(CONFIG_UMP)) || defined(CONFIG_DMA_SHARED_BUFFER_USES_KDS)
static void add_kds_resource(struct kds_resource *kds_res, struct kds_resource **kds_resources, u32 *kds_res_count, unsigned long *kds_access_bitmap, mali_bool exclusive)
{
	u32 i;

	for (i = 0; i < *kds_res_count; i++) {
		/* Duplicate resource, ignore */
		if (kds_resources[i] == kds_res)
			return;
	}

	kds_resources[*kds_res_count] = kds_res;
	if (exclusive)
		set_bit(*kds_res_count, kds_access_bitmap);
	(*kds_res_count)++;
}
#endif

/*
 * Set up external resources needed by this job.
 *
 * jctx.lock must be held when this is called.
 */

static mali_error kbase_jd_pre_external_resources(kbase_jd_atom *katom, const base_jd_atom_v2 *user_atom)
{
	mali_error err_ret_val = MALI_ERROR_FUNCTION_FAILED;
	u32 res_no;
#ifdef CONFIG_KDS
	u32 kds_res_count = 0;
	struct kds_resource **kds_resources = NULL;
	unsigned long *kds_access_bitmap = NULL;
#endif				/* CONFIG_KDS */

	KBASE_DEBUG_ASSERT(katom);
	KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES);

	/* no resources encoded, early out */
	if (!katom->nr_extres)
		return MALI_ERROR_FUNCTION_FAILED;

	katom->extres = kmalloc(sizeof(base_external_resource) * katom->nr_extres, GFP_KERNEL);
	if (NULL == katom->extres) {
		err_ret_val = MALI_ERROR_OUT_OF_MEMORY;
		goto early_err_out;
	}

	if (copy_from_user(katom->extres, get_compat_pointer(&user_atom->extres_list), sizeof(base_external_resource) * katom->nr_extres) != 0) {
		err_ret_val = MALI_ERROR_FUNCTION_FAILED;
		goto early_err_out;
	}
#ifdef CONFIG_KDS
	/* assume we have to wait for all */
	KBASE_DEBUG_ASSERT(0 != katom->nr_extres);
	kds_resources = kmalloc(sizeof(struct kds_resource *) * katom->nr_extres, GFP_KERNEL);

	if (NULL == kds_resources) {
		err_ret_val = MALI_ERROR_OUT_OF_MEMORY;
		goto early_err_out;
	}

	KBASE_DEBUG_ASSERT(0 != katom->nr_extres);
	kds_access_bitmap = kzalloc(sizeof(unsigned long) * ((katom->nr_extres + BITS_PER_LONG - 1) / BITS_PER_LONG), GFP_KERNEL);

	if (NULL == kds_access_bitmap) {
		err_ret_val = MALI_ERROR_OUT_OF_MEMORY;
		goto early_err_out;
	}
#endif				/* CONFIG_KDS */

#if defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG)
	/* need to keep the GPU VM locked while we set up UMM buffers */
	/* Lock also used in debug mode just for lock order checking */
	kbase_gpu_vm_lock(katom->kctx);
#endif				/* defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG) */

	for (res_no = 0; res_no < katom->nr_extres; res_no++) {
		base_external_resource *res;
		kbase_va_region *reg;

		res = &katom->extres[res_no];
		reg = kbase_region_tracker_find_region_enclosing_address(katom->kctx, res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE);
		/* did we find a matching region object? */
		if (NULL == reg) {
			/* roll back */
			goto failed_loop;
		}

		/* decide what needs to happen for this resource */
		switch (reg->imported_type) {
		case BASE_TMEM_IMPORT_TYPE_UMP:
			{
#if defined(CONFIG_KDS) && defined(CONFIG_UMP)
				struct kds_resource *kds_res;
				kds_res = ump_dd_kds_resource_get(reg->imported_metadata.ump_handle);
				if (kds_res)
					add_kds_resource(kds_res, kds_resources, &kds_res_count, kds_access_bitmap, katom->extres[res_no].ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE);
#endif				/*defined(CONFIG_KDS) && defined(CONFIG_UMP) */
				break;
			}
#ifdef CONFIG_DMA_SHARED_BUFFER
		case BASE_TMEM_IMPORT_TYPE_UMM:
			{
#ifdef CONFIG_DMA_SHARED_BUFFER_USES_KDS
				struct kds_resource *kds_res;
				kds_res = get_dma_buf_kds_resource(reg->imported_metadata.umm.dma_buf);
				if (kds_res)
					add_kds_resource(kds_res, kds_resources, &kds_res_count, kds_access_bitmap, katom->extres[res_no].ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE);
#endif
				reg->imported_metadata.umm.current_mapping_usage_count++;
				if (1 == reg->imported_metadata.umm.current_mapping_usage_count) {
					/* use a local variable to not pollute err_ret_val
					 * with a potential success value as some other gotos depend
					 * on the default error code stored in err_ret_val */
					mali_error tmp;
					tmp = kbase_jd_umm_map(katom->kctx, reg);
					if (MALI_ERROR_NONE != tmp) {
						/* failed to map this buffer, roll back */
						err_ret_val = tmp;
						reg->imported_metadata.umm.current_mapping_usage_count--;
						goto failed_loop;
					}
				}
				break;
			}
#endif
		default:
			goto failed_loop;
		}
	}
	/* successfully parsed the extres array */
#if defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG)
	/* drop the vm lock before we call into kds */
	/* Lock also used in debug mode just for lock order checking */
	kbase_gpu_vm_unlock(katom->kctx);
#endif				/* defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG) */

#ifdef CONFIG_KDS
	if (kds_res_count) {
		int wait_failed;
		/* We have resources to wait for with kds */
		katom->kds_dep_satisfied = MALI_FALSE;

		wait_failed = kds_async_waitall(&katom->kds_rset,
										&katom->kctx->jctx.kds_cb,
										katom,
										NULL,
										kds_res_count,
										kds_access_bitmap,
										kds_resources);
		if (wait_failed) {
			goto failed_kds_setup;
		} else {
			kbase_jd_kds_waiters_add( katom );
		}
	} else {
		/* Nothing to wait for, so kds dep met */
		katom->kds_dep_satisfied = MALI_TRUE;
	}
	kfree(kds_resources);
	kfree(kds_access_bitmap);
#endif				/* CONFIG_KDS */

	/* all done OK */
	return MALI_ERROR_NONE;

/* error handling section */

#ifdef CONFIG_KDS
 failed_kds_setup:

#if defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG)
	/* lock before we unmap */
	/* Lock also used in debug mode just for lock order checking */
	kbase_gpu_vm_lock(katom->kctx);
#endif				/* defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG) */
#endif				/* CONFIG_KDS */

 failed_loop:
#ifdef CONFIG_DMA_SHARED_BUFFER
	/* undo the loop work */
	while (res_no-- > 0) {
		base_external_resource *res;
		kbase_va_region *reg;

		res = &katom->extres[res_no];
		reg = kbase_region_tracker_find_region_enclosing_address(katom->kctx, res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE);
		/* if reg wasn't found then it has been freed when we set up kds */
		if (reg && reg->imported_type == BASE_TMEM_IMPORT_TYPE_UMM) {
			reg->imported_metadata.umm.current_mapping_usage_count--;
			if (0 == reg->imported_metadata.umm.current_mapping_usage_count)
				kbase_jd_umm_unmap(katom->kctx, reg);
		}
	}
#endif				/* CONFIG_DMA_SHARED_BUFFER */
#if defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG)
	/* Lock also used in debug mode just for lock order checking */
	kbase_gpu_vm_unlock(katom->kctx);
#endif				/* defined(CONFIG_DMA_SHARED_BUFFER) || defined(CONFIG_MALI_DEBUG) */

 early_err_out:
	kfree(katom->extres);
	katom->extres = NULL;
#ifdef CONFIG_KDS
	kfree(kds_resources);
	kfree(kds_access_bitmap);
#endif				/* CONFIG_KDS */
	return err_ret_val;
}

STATIC INLINE void jd_resolve_dep(struct list_head *out_list, kbase_jd_atom *katom, u8 d)
{
	u8 other_d = !d;

	while (!list_empty(&katom->dep_head[d])) {
		kbase_jd_atom *dep_atom = list_entry(katom->dep_head[d].next, kbase_jd_atom, dep_item[d]);
		list_del(katom->dep_head[d].next);

		dep_atom->dep_atom[d] = NULL;

		if (katom->event_code != BASE_JD_EVENT_DONE) {
			/* Atom failed, so remove the other dependencies and immediately fail the atom */
			if (dep_atom->dep_atom[other_d]) {
				list_del(&dep_atom->dep_item[other_d]);
				dep_atom->dep_atom[other_d] = NULL;
			}
#ifdef CONFIG_KDS
			if (!dep_atom->kds_dep_satisfied) {
				/* Just set kds_dep_satisfied to true. If the callback happens after this then it will early out and
				 * do nothing. If the callback doesn't happen then kbase_jd_post_external_resources will clean up
				 */
				dep_atom->kds_dep_satisfied = MALI_TRUE;
			}
#endif

			dep_atom->event_code = katom->event_code;
			KBASE_DEBUG_ASSERT(dep_atom->status != KBASE_JD_ATOM_STATE_UNUSED);
			dep_atom->status = KBASE_JD_ATOM_STATE_COMPLETED;

			list_add(&dep_atom->dep_item[0], out_list);
		} else if (!dep_atom->dep_atom[other_d]) {
#ifdef CONFIG_KDS
			if (dep_atom->kds_dep_satisfied)
#endif
				list_add(&dep_atom->dep_item[0], out_list);
		}
	}
}

KBASE_EXPORT_TEST_API(jd_resolve_dep)

/*
 * Perform the necessary handling of an atom that has finished running
 * on the GPU.
 *
 * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller
 * is responsible for calling kbase_finish_soft_job *before* calling this function.
 *
 * The caller must hold the kbase_jd_context.lock.
 */
mali_bool jd_done_nolock(kbase_jd_atom *katom)
{
	struct kbase_context *kctx = katom->kctx;
	kbase_device *kbdev = kctx->kbdev;
	struct list_head completed_jobs;
	struct list_head runnable_jobs;
	mali_bool need_to_try_schedule_context = MALI_FALSE;
	int i;

	INIT_LIST_HEAD(&completed_jobs);
	INIT_LIST_HEAD(&runnable_jobs);

	KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED);

	/* This is needed in case an atom is failed due to being invalid, this
	 * can happen *before* the jobs that the atom depends on have completed */
	for (i = 0; i < 2; i++) {
		if (katom->dep_atom[i]) {
			list_del(&katom->dep_item[i]);
			katom->dep_atom[i] = NULL;
		}
	}

	/* With PRLAM-10817 the last tile of a fragment job being soft-stopped can fail with
	 * BASE_JD_EVENT_TILE_RANGE_FAULT.
	 *
	 * So here if the fragment job failed with TILE_RANGE_FAULT and it has been soft-stopped, then we promote the
	 * error code to BASE_JD_EVENT_DONE
	 */

	if ( kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10817) && katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT ) {
		if ( ( katom->core_req & BASE_JD_REQ_FS ) && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) ) {
			/* Promote the failure to job done */
			katom->event_code = BASE_JD_EVENT_DONE;
			katom->atom_flags = katom->atom_flags & (~KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED);
		}
	}

	katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
	list_add_tail(&katom->dep_item[0], &completed_jobs);

	while (!list_empty(&completed_jobs)) {
		katom = list_entry(completed_jobs.prev, kbase_jd_atom, dep_item[0]);
		list_del(completed_jobs.prev);

		KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED);

		for (i = 0; i < 2; i++)
			jd_resolve_dep(&runnable_jobs, katom, i);

		while (!list_empty(&runnable_jobs)) {
			kbase_jd_atom *node = list_entry(runnable_jobs.prev, kbase_jd_atom, dep_item[0]);
			list_del(runnable_jobs.prev);

			KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED);

			if (katom->event_code == BASE_JD_EVENT_DONE) {
				need_to_try_schedule_context |= jd_run_atom(node);
			} else {
				node->event_code = katom->event_code;
				node->status = KBASE_JD_ATOM_STATE_COMPLETED;

				if (node->core_req & BASE_JD_REQ_SOFT_JOB) {
					/* If this is a fence wait then remove it from the list of sync waiters. */
					if ( BASE_JD_REQ_SOFT_FENCE_WAIT == node->core_req )
						list_del(&node->dep_item[0]);

					kbase_finish_soft_job(node);
				}
			}

			if (node->status == KBASE_JD_ATOM_STATE_COMPLETED)
				list_add_tail(&node->dep_item[0], &completed_jobs);
		}

		if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES)
			kbase_jd_post_external_resources(katom);

		kbase_event_post(kctx, katom);

		/* Decrement and check the TOTAL number of jobs. This includes
		 * those not tracked by the scheduler: 'not ready to run' and
		 * 'dependency-only' jobs. */
		if (--kctx->jctx.job_nr == 0)
			wake_up(&kctx->jctx.zero_jobs_wait);	/* All events are safely queued now, and we can signal any waiter
								 * that we've got no more jobs (so we can be safely terminated) */
	}

	return need_to_try_schedule_context;
}

KBASE_EXPORT_TEST_API(jd_done_nolock)

#ifdef CONFIG_GPU_TRACEPOINTS
enum {
	CORE_REQ_DEP_ONLY,
	CORE_REQ_SOFT,
	CORE_REQ_COMPUTE,
	CORE_REQ_FRAGMENT,
	CORE_REQ_VERTEX,
	CORE_REQ_TILER,
	CORE_REQ_FRAGMENT_VERTEX,
	CORE_REQ_FRAGMENT_VERTEX_TILER,
	CORE_REQ_FRAGMENT_TILER,
	CORE_REQ_VERTEX_TILER,
	CORE_REQ_UNKNOWN
};
static const char * const core_req_strings[] = {
	"Dependency Only Job",
	"Soft Job",
	"Compute Shader Job",
	"Fragment Shader Job",
	"Vertex/Geometry Shader Job",
	"Tiler Job",
	"Fragment Shader + Vertex/Geometry Shader Job",
	"Fragment Shader + Vertex/Geometry Shader Job + Tiler Job",
	"Fragment Shader + Tiler Job",
	"Vertex/Geometry Shader Job + Tiler Job",
	"Unknown Job"
};
static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req)
{
	if (core_req & BASE_JD_REQ_SOFT_JOB)
		return core_req_strings[CORE_REQ_SOFT];
	if (core_req & BASE_JD_REQ_ONLY_COMPUTE)
		return core_req_strings[CORE_REQ_COMPUTE];
	switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) {
	case BASE_JD_REQ_DEP:
		return core_req_strings[CORE_REQ_DEP_ONLY];
	case BASE_JD_REQ_FS:
		return core_req_strings[CORE_REQ_FRAGMENT];
	case BASE_JD_REQ_CS:
		return core_req_strings[CORE_REQ_VERTEX];
	case BASE_JD_REQ_T:
		return core_req_strings[CORE_REQ_TILER];
	case (BASE_JD_REQ_FS | BASE_JD_REQ_CS):
		return core_req_strings[CORE_REQ_FRAGMENT_VERTEX];
	case (BASE_JD_REQ_FS | BASE_JD_REQ_T):
		return core_req_strings[CORE_REQ_FRAGMENT_TILER];
	case (BASE_JD_REQ_CS | BASE_JD_REQ_T):
		return core_req_strings[CORE_REQ_VERTEX_TILER];
	case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T):
		return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER];
	}
	return core_req_strings[CORE_REQ_UNKNOWN];
}
#endif

static mali_bool jd_submit_atom(kbase_context *kctx, const base_jd_atom_v2 *user_atom)
{
	kbase_jd_context *jctx = &kctx->jctx;
	kbase_jd_atom *katom;
	base_jd_core_req core_req;
	base_atom_id atom_number = user_atom->atom_number;
	int queued = 0;
	int i;
	mali_bool ret;

	katom = &jctx->atoms[atom_number];

	mutex_lock(&jctx->lock);
	while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) {
		/* Atom number is already in use, wait for the atom to complete */
		mutex_unlock(&jctx->lock);
		
		/* This thread will wait for the atom to complete. Due to thread scheduling we are not sure that
		 * the other thread that owns the atom will also schedule the context, so we force the scheduler
		 * to be active and hence eventually schedule this context at some point later. 
		 */
		kbasep_js_try_schedule_head_ctx(kctx->kbdev);
		if (wait_event_killable(katom->completed, katom->status == KBASE_JD_ATOM_STATE_UNUSED)) {
			/* We're being killed so the result code doesn't really matter */
			return MALI_FALSE;
		}
		mutex_lock(&jctx->lock);
	}

	/* Update the TOTAL number of jobs. This includes those not tracked by
	 * the scheduler: 'not ready to run' and 'dependency-only' jobs. */
	jctx->job_nr++;

	core_req = user_atom->core_req;

	katom->udata = user_atom->udata;
	katom->kctx = kctx;
	katom->nr_extres = user_atom->nr_extres;
	katom->extres = NULL;
	katom->device_nr = user_atom->device_nr;
	katom->affinity = 0;
	katom->jc = user_atom->jc;
	katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
	katom->core_req = core_req;
	katom->nice_prio = user_atom->prio;
	katom->atom_flags = 0;

#ifdef CONFIG_KDS
	/* Start by assuming that the KDS dependencies are satisfied,
	 * kbase_jd_pre_external_resources will correct this if there are dependencies */
	katom->kds_dep_satisfied = MALI_TRUE;
	katom->kds_rset = NULL;
#endif				/* CONFIG_KDS */

	/* Add dependencies */
	for (i = 0; i < 2; i++) {
		int dep_atom_number = user_atom->pre_dep[i];
		katom->dep_atom[i] = NULL;
		if (dep_atom_number) {
			kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number];

			if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
				if (dep_atom->event_code != BASE_JD_EVENT_DONE) {
					if (i == 1 && katom->dep_atom[0]) {
						/* Remove the previous dependency */
						list_del(&katom->dep_item[0]);
						katom->dep_atom[0] = NULL;
					}
					/* Atom has completed, propagate the error code if any */
					katom->event_code = dep_atom->event_code;
					katom->status = KBASE_JD_ATOM_STATE_QUEUED;
					ret = jd_done_nolock(katom);
					goto out;
				}
			} else {
				/* Atom is in progress, add this atom to the list */
				list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]);
				katom->dep_atom[i] = dep_atom;
				queued = 1;
			}
		}
	}

	/* These must occur after the above loop to ensure that an atom that
	 * depends on a previous atom with the same number behaves as expected */
	katom->event_code = BASE_JD_EVENT_DONE;
	katom->status = KBASE_JD_ATOM_STATE_QUEUED;

	/* Reject atoms with job chain = NULL, as these cause issues with soft-stop */
	if (0 == katom->jc && (katom->core_req & BASEP_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP)
	{
		KBASE_DEBUG_PRINT_WARN(KBASE_JD, "Rejecting atom with jc = NULL");
		katom->event_code = BASE_JD_EVENT_JOB_INVALID;
		ret = jd_done_nolock(katom);
		goto out;
	}

	/*
	 * If the priority is increased we need to check the caller has security caps to do this, if
	 * priority is decreased then this is ok as the result will have no negative impact on other
	 * processes running.
	 */
	if (0 > katom->nice_prio) {
		mali_bool access_allowed;
		access_allowed = kbase_security_has_capability(kctx, KBASE_SEC_MODIFY_PRIORITY, KBASE_SEC_FLAG_NOAUDIT);
		if (!access_allowed) {
			/* For unprivileged processes - a negative priority is interpreted as zero */
			katom->nice_prio = 0;
		}
	}

	/* Scale priority range to use NICE range */
	if (katom->nice_prio) {
		/* Remove sign for calculation */
		int nice_priority = katom->nice_prio + 128;
		/* Fixed point maths to scale from ..255 to 0..39 (NICE range with +20 offset) */
		katom->nice_prio = (((20 << 16) / 128) * nice_priority) >> 16;
	}

	if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
		/* handle what we need to do to access the external resources */
		if (MALI_ERROR_NONE != kbase_jd_pre_external_resources(katom, user_atom)) {
			/* setup failed (no access, bad resource, unknown resource types, etc.) */
			katom->event_code = BASE_JD_EVENT_JOB_INVALID;
			ret = jd_done_nolock(katom);
			goto out;
		}
	}

	/* Initialize the jobscheduler policy for this atom. Function will
	 * return error if the atom is malformed.
	 *
	 * Soft-jobs never enter the job scheduler but have their own initialize method.
	 *
	 * If either fail then we immediately complete the atom with an error.
	 */
	if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) {
		kbasep_js_policy *js_policy = &(kctx->kbdev->js_data.policy);
		if (MALI_ERROR_NONE != kbasep_js_policy_init_job(js_policy, kctx, katom)) {
			katom->event_code = BASE_JD_EVENT_JOB_INVALID;
			ret = jd_done_nolock(katom);
			goto out;
		}
	} else {
		/* Soft-job */
		if (MALI_ERROR_NONE != kbase_prepare_soft_job(katom)) {
			katom->event_code = BASE_JD_EVENT_JOB_INVALID;
			ret = jd_done_nolock(katom);
			goto out;
		}
	}

#ifdef CONFIG_GPU_TRACEPOINTS
	katom->work_id = atomic_inc_return(&jctx->work_id);
	trace_gpu_job_enqueue((u32)kctx, katom->work_id, kbasep_map_core_reqs_to_string(katom->core_req));
#endif

	if (queued) {
		ret = MALI_FALSE;
		goto out;
	}
#ifdef CONFIG_KDS
	if (!katom->kds_dep_satisfied) {
		/* Queue atom due to KDS dependency */
		ret = MALI_FALSE;
		goto out;
	}
#endif				/* CONFIG_KDS */

	if (katom->core_req & BASE_JD_REQ_SOFT_JOB) {
		if (kbase_process_soft_job(katom) == 0) {
			kbase_finish_soft_job(katom);
			ret = jd_done_nolock(katom);
			goto out;
		}
		/* The job has not yet completed */
		list_add_tail(&katom->dep_item[0], &kctx->waiting_soft_jobs);
		ret = MALI_FALSE;
	} else if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
		katom->status = KBASE_JD_ATOM_STATE_IN_JS;
		ret = kbasep_js_add_job(kctx, katom);
	} else {
		/* This is a pure dependency. Resolve it immediately */
		ret = jd_done_nolock(katom);
	}

 out:
	mutex_unlock(&jctx->lock);
	return ret;
}

mali_error kbase_jd_submit(kbase_context *kctx, const kbase_uk_job_submit *submit_data)
{
	mali_error err = MALI_ERROR_NONE;
	int i;
	mali_bool need_to_try_schedule_context = MALI_FALSE;
	kbase_device *kbdev;
	void *user_addr;

	/*
	 * kbase_jd_submit isn't expected to fail and so all errors with the jobs
	 * are reported by immediately falling them (through event system)
	 */
	kbdev = kctx->kbdev;

	beenthere("%s", "Enter");

	if ((kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) != 0) {
		KBASE_DEBUG_PRINT_ERROR(KBASE_JD, "Attempt to submit to a context that has SUBMIT_DISABLED set on it");
		return MALI_ERROR_FUNCTION_FAILED;
	}

	if (submit_data->stride != sizeof(base_jd_atom_v2)) {
		KBASE_DEBUG_PRINT_ERROR(KBASE_JD, "Stride passed to job_submit doesn't match kernel");
		return MALI_ERROR_FUNCTION_FAILED;
	}

	user_addr = get_compat_pointer(&submit_data->addr);

	KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(submit_data->nr_atoms, &kctx->timeline.jd_atoms_in_flight));

	for (i = 0; i < submit_data->nr_atoms; i++) {
		base_jd_atom_v2 user_atom;

		if (copy_from_user(&user_atom, user_addr, sizeof(user_atom)) != 0) {
			err = MALI_ERROR_FUNCTION_FAILED;
			KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(submit_data->nr_atoms - i, &kctx->timeline.jd_atoms_in_flight));
			break;
		}

		user_addr = (void *)((uintptr_t) user_addr + submit_data->stride);

		need_to_try_schedule_context |= jd_submit_atom(kctx, &user_atom);
	}

	if (need_to_try_schedule_context)
		kbasep_js_try_schedule_head_ctx(kbdev);

	return err;
}

KBASE_EXPORT_TEST_API(kbase_jd_submit)

static void kbasep_jd_cacheclean(kbase_device *kbdev)
{
	/* Limit the number of loops to avoid a hang if the interrupt is missed */
	u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;

	mutex_lock(&kbdev->cacheclean_lock);

	/* use GPU_COMMAND completion solution */
	/* clean & invalidate the caches */
	KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CLEAN_INV_CACHES, NULL);

	/* wait for cache flush to complete before continuing */
	while (--max_loops && (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & CLEAN_CACHES_COMPLETED) == 0)
		;

	/* clear the CLEAN_CACHES_COMPLETED irq */
	KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, CLEAN_CACHES_COMPLETED);
	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), CLEAN_CACHES_COMPLETED, NULL);
	KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.state != KBASE_INSTR_STATE_CLEANING,
	    "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang.");

	mutex_unlock(&kbdev->cacheclean_lock);
}

/**
 * This function:
 * - requeues the job from the runpool (if it was soft-stopped/removed from NEXT registers)
 * - removes it from the system if it finished/failed/was cancelled.
 * - resolves dependencies to add dependent jobs to the context, potentially starting them if necessary (which may add more references to the context)
 * - releases the reference to the context from the no-longer-running job.
 * - Handles retrying submission outside of IRQ context if it failed from within IRQ context.
 */
static void jd_done_worker(struct work_struct *data)
{
	kbase_jd_atom *katom = container_of(data, kbase_jd_atom, work);
	kbase_jd_context *jctx;
	kbase_context *kctx;
	kbasep_js_kctx_info *js_kctx_info;
	kbasep_js_policy *js_policy;
	kbase_device *kbdev;
	kbasep_js_device_data *js_devdata;
	u64 cache_jc = katom->jc;
	kbasep_js_atom_retained_state katom_retained_state;

	/* Soft jobs should never reach this function */
	KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);

	kctx = katom->kctx;
	jctx = &kctx->jctx;
	kbdev = kctx->kbdev;
	js_kctx_info = &kctx->jctx.sched_info;

	js_devdata = &kbdev->js_data;
	js_policy = &kbdev->js_data.policy;

	KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0);
	/*
	 * Begin transaction on JD context and JS context
	 */
	mutex_lock(&jctx->lock);
	mutex_lock(&js_kctx_info->ctx.jsctx_mutex);

	/* This worker only gets called on contexts that are scheduled *in*. This is
	 * because it only happens in response to an IRQ from a job that was
	 * running.
	 */
	KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled != MALI_FALSE);

	if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) && katom->event_code != BASE_JD_EVENT_DONE && !(katom->event_code & BASE_JD_SW_EVENT))
		kbasep_jd_cacheclean(kbdev);  /* cache flush when jobs complete with non-done codes */
	else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) {
		if (kbdev->gpu_props.num_core_groups > 1 && 
		    !(katom->affinity & kbdev->gpu_props.props.coherency_info.group[0].core_mask) &&
		    (katom->affinity & kbdev->gpu_props.props.coherency_info.group[1].core_mask)) {
			KBASE_DEBUG_PRINT_INFO(KBASE_JD, "JD: Flushing cache due to PRLAM-10676\n");
			kbasep_jd_cacheclean(kbdev);
		}
	}

	if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969)            &&
	    (katom->core_req & BASE_JD_REQ_FS)                        &&
	    katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT       &&
	    (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) &&
	    !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)){
		KBASE_DEBUG_PRINT_INFO(KBASE_JD,
				       " Soft-stopped fragment shader job got a TILE_RANGE_FAULT." \
				       "Possible HW issue, trying SW workaround \n " );
		if (kbasep_10969_workaround_clamp_coordinates(katom)){
			/* The job had a TILE_RANGE_FAULT after was soft-stopped.
			 * Due to an HW issue we try to execute the job
			 * again.
			 */
			KBASE_DEBUG_PRINT_INFO(KBASE_JD, " Clamping has been executed, try to rerun the job \n" );
			katom->event_code = BASE_JD_EVENT_STOPPED;
			katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN;

			/* The atom will be requeued, but requeing does not submit more
			 * jobs. If this was the last job, we must also ensure that more
			 * jobs will be run on slot 0 - this is a Fragment job. */
			kbasep_js_set_job_retry_submit_slot(katom, 0);
		}
	}

	/* If job was rejected due to BASE_JD_EVENT_PM_EVENT but was not
	 * specifically targeting core group 1, then re-submit targeting core
	 * group 0 */
	if (katom->event_code == BASE_JD_EVENT_PM_EVENT && !(katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) {
		katom->event_code = BASE_JD_EVENT_STOPPED;
		/* Don't need to worry about any previously set retry-slot - it's
		 * impossible for it to have been set previously, because we guarantee
		 * kbase_jd_done() was called with done_code==0 on this atom */
		kbasep_js_set_job_retry_submit_slot(katom, 1);
	}

	if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316))
		kbase_as_poking_timer_release_atom(kbdev, kctx, katom);

	/* Release cores this job was using (this might power down unused cores, and
	 * cause extra latency if a job submitted here - such as depenedent jobs -
	 * would use those cores) */
	kbasep_js_job_check_deref_cores(kbdev, katom);

	/* Retain state before the katom disappears */
	kbasep_js_atom_retained_state_copy(&katom_retained_state, katom);

	if (!kbasep_js_has_atom_finished(&katom_retained_state)) {
		unsigned long flags;
		/* Requeue the atom on soft-stop / removed from NEXT registers */
		KBASE_DEBUG_PRINT_INFO(KBASE_JM, "JS: Soft Stopped/Removed from next on Ctx %p; Requeuing", kctx);

		mutex_lock(&js_devdata->runpool_mutex);
		kbasep_js_clear_job_retry_submit(katom);

		KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, katom));
		spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
		kbasep_js_policy_enqueue_job(js_policy, katom);
		spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);

		/* A STOPPED/REMOVED job must cause a re-submit to happen, in case it
		 * was the last job left. Crucially, work items on work queues can run
		 * out of order e.g. on different CPUs, so being able to submit from
		 * the IRQ handler is not a good indication that we don't need to run
		 * jobs; the submitted job could be processed on the work-queue
		 * *before* the stopped job, even though it was submitted after. */
		{
			int tmp;
			KBASE_DEBUG_ASSERT(kbasep_js_get_atom_retry_submit_slot(&katom_retained_state, &tmp) != MALI_FALSE);
			CSTD_UNUSED(tmp);
		}

		mutex_unlock(&js_devdata->runpool_mutex);
		mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
	} else {
		/* Remove the job from the system for all other reasons */
		mali_bool need_to_try_schedule_context;

		kbasep_js_remove_job(kbdev, kctx, katom);
		mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
		/* jd_done_nolock() requires the jsctx_mutex lock to be dropped */

		need_to_try_schedule_context = jd_done_nolock(katom);

		/* This ctx is already scheduled in, so return value guarenteed FALSE */
		KBASE_DEBUG_ASSERT(need_to_try_schedule_context == MALI_FALSE);
	}
	/* katom may have been freed now, do not use! */

	/*
	 * Transaction complete
	 */
	mutex_unlock(&jctx->lock);

	/* Job is now no longer running, so can now safely release the context
	 * reference, and handle any actions that were logged against the atom's retained state */
	kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state);

	KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0);
}

/**
 * Work queue job cancel function
 * Only called as part of 'Zapping' a context (which occurs on termination)
 * Operates serially with the jd_done_worker() on the work queue.
 *
 * This can only be called on contexts that aren't scheduled.
 *
 * @note We don't need to release most of the resources that would occur on
 * kbase_jd_done() or jd_done_worker(), because the atoms here must not be
 * running (by virtue of only being called on contexts that aren't
 * scheduled). The only resources that are an exception to this are:
 * - those held by kbasep_js_job_check_ref_cores(), because these resources are
 *   held for non-running atoms as well as running atoms.
 */
static void jd_cancel_worker(struct work_struct *data)
{
	kbase_jd_atom *katom = container_of(data, kbase_jd_atom, work);
	kbase_jd_context *jctx;
	kbase_context *kctx;
	kbasep_js_kctx_info *js_kctx_info;
	mali_bool need_to_try_schedule_context;
	kbase_device *kbdev;

	/* Soft jobs should never reach this function */
	KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);

	kctx = katom->kctx;
	kbdev = kctx->kbdev;
	jctx = &kctx->jctx;
	js_kctx_info = &kctx->jctx.sched_info;

	KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0);

	/* This only gets called on contexts that are scheduled out. Hence, we must
	 * make sure we don't de-ref the number of running jobs (there aren't
	 * any), nor must we try to schedule out the context (it's already
	 * scheduled out).
	 */
	KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled == MALI_FALSE);

	/* Release cores this job was using (this might power down unused cores) */
	kbasep_js_job_check_deref_cores(kctx->kbdev, katom);

	/* Scheduler: Remove the job from the system */
	mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
	kbasep_js_remove_cancelled_job(kbdev, kctx, katom);
	mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);

	mutex_lock(&jctx->lock);

	need_to_try_schedule_context = jd_done_nolock(katom);
	/* Because we're zapping, we're not adding any more jobs to this ctx, so no need to
	 * schedule the context. There's also no need for the jsctx_mutex to have been taken
	 * around this too. */
	KBASE_DEBUG_ASSERT(need_to_try_schedule_context == MALI_FALSE);

	/* katom may have been freed now, do not use! */
	mutex_unlock(&jctx->lock);

}

/**
 * @brief Complete a job that has been removed from the Hardware
 *
 * This must be used whenever a job has been removed from the Hardware, e.g.:
 * - An IRQ indicates that the job finished (for both error and 'done' codes)
 * - The job was evicted from the JSn_HEAD_NEXT registers during a Soft/Hard stop.
 *
 * Some work is carried out immediately, and the rest is deferred onto a workqueue
 *
 * This can be called safely from atomic context.
 *
 * The caller must hold kbasep_js_device_data::runpool_irq::lock
 *
 */
void kbase_jd_done(kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp,
                   kbasep_js_atom_done_code done_code)
{
	kbase_context *kctx;
	kbase_device *kbdev;
	KBASE_DEBUG_ASSERT(katom);
	kctx = katom->kctx;
	KBASE_DEBUG_ASSERT(kctx);
	kbdev = kctx->kbdev;
	KBASE_DEBUG_ASSERT(kbdev);

	if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT)
		katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;

	kbase_timeline_job_slot_done(kbdev, kctx, katom, slot_nr, done_code);

	KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0);

	kbasep_js_job_done_slot_irq(katom, slot_nr, end_timestamp, done_code);

	katom->slot_nr = slot_nr;

	KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
	INIT_WORK(&katom->work, jd_done_worker);
	queue_work(kctx->jctx.job_done_wq, &katom->work);
}

KBASE_EXPORT_TEST_API(kbase_jd_done)

void kbase_jd_cancel(kbase_device *kbdev, kbase_jd_atom *katom)
{
	kbase_context *kctx;
	kbasep_js_kctx_info *js_kctx_info;
	KBASE_DEBUG_ASSERT(NULL != kbdev);
	KBASE_DEBUG_ASSERT(NULL != katom);
	kctx = katom->kctx;
	KBASE_DEBUG_ASSERT(NULL != kctx);

	js_kctx_info = &kctx->jctx.sched_info;

	KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0);

	/* This should only be done from a context that is not scheduled */
	KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled == MALI_FALSE);

	katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;

	KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
	INIT_WORK(&katom->work, jd_cancel_worker);
	queue_work(kctx->jctx.job_done_wq, &katom->work);
}

typedef struct zap_reset_data {
	/* The stages are:
	 * 1. The timer has never been called
	 * 2. The zap has timed out, all slots are soft-stopped - the GPU reset will happen.
	 *    The GPU has been reset when kbdev->reset_waitq is signalled
	 *
	 * (-1 - The timer has been cancelled)
	 */
	int stage;
	kbase_device *kbdev;
	struct hrtimer timer;
	spinlock_t lock;
} zap_reset_data;

static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer)
{
	zap_reset_data *reset_data = container_of(timer, zap_reset_data, timer);
	kbase_device *kbdev = reset_data->kbdev;
	unsigned long flags;

	spin_lock_irqsave(&reset_data->lock, flags);

	if (reset_data->stage == -1)
		goto out;

	if (kbase_prepare_to_reset_gpu(kbdev)) {
		KBASE_DEBUG_PRINT_ERROR(KBASE_JD, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", ZAP_TIMEOUT);
		kbase_reset_gpu(kbdev);
	}

	reset_data->stage = 2;

 out:
	spin_unlock_irqrestore(&reset_data->lock, flags);

	return HRTIMER_NORESTART;
}

void kbase_jd_zap_context(kbase_context *kctx)
{
	kbase_jd_atom *katom;
#ifdef CONFIG_KDS
	struct list_head *entry;
#endif
	kbase_device *kbdev;
	zap_reset_data reset_data;
	unsigned long flags;

	KBASE_DEBUG_ASSERT(kctx);

	kbdev = kctx->kbdev;

	KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u);
	kbase_job_zap_context(kctx);

	mutex_lock(&kctx->jctx.lock);

	/*
	 * While holding the kbase_jd_context lock clean up jobs which are known to kbase but are
	 * queued outside the job scheduler.
	 */

	while (!list_empty(&kctx->waiting_soft_jobs)) {
		katom = list_first_entry(&kctx->waiting_soft_jobs,
					kbase_jd_atom,
					dep_item[0]);
		list_del(&katom->dep_item[0]);
		kbase_cancel_soft_job(katom);
	}

#ifdef CONFIG_KDS

	/* For each job waiting on a kds resource, cancel the wait and force the job to
	 * complete early, this is done so that we don't leave jobs outstanding waiting
	 * on kds resources which may never be released when contexts are zapped, resulting
	 * in a hang.
	 *
	 * Note that we can safely iterate over the list as the kbase_jd_context lock is held,
	 * this prevents items being removed when calling job_done_nolock in kbase_cancel_kds_wait_job.
	 */

	list_for_each( entry, &kctx->waiting_kds_resource) {
		katom = list_entry(entry, kbase_jd_atom, node);

		kbase_cancel_kds_wait_job(katom);
	}
#endif

	mutex_unlock(&kctx->jctx.lock);

	hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	reset_data.timer.function = zap_timeout_callback;

	spin_lock_init(&reset_data.lock);

	reset_data.kbdev = kbdev;
	reset_data.stage = 1;

	hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT), HRTIMER_MODE_REL);

	/* Wait for all jobs to finish, and for the context to be not-scheduled
	 * (due to kbase_job_zap_context(), we also guarentee it's not in the JS
	 * policy queue either */
	wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0);
	wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, kctx->jctx.sched_info.ctx.is_scheduled == MALI_FALSE);

	spin_lock_irqsave(&reset_data.lock, flags);
	if (reset_data.stage == 1) {
		/* The timer hasn't run yet - so cancel it */
		reset_data.stage = -1;
	}
	spin_unlock_irqrestore(&reset_data.lock, flags);

	hrtimer_cancel(&reset_data.timer);

	if (reset_data.stage == 2) {
		/* The reset has already started.
		 * Wait for the reset to complete
		 */
		wait_event(kbdev->reset_wait, atomic_read(&kbdev->reset_gpu) == KBASE_RESET_GPU_NOT_PENDING);
	}
	destroy_hrtimer_on_stack(&reset_data.timer);

	KBASE_DEBUG_PRINT_INFO(KBASE_JM, "Zap: Finished Context %p", kctx);

	/* Ensure that the signallers of the waitqs have finished */
	mutex_lock(&kctx->jctx.lock);
	mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
	mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
	mutex_unlock(&kctx->jctx.lock);
}

KBASE_EXPORT_TEST_API(kbase_jd_zap_context)

mali_error kbase_jd_init(kbase_context *kctx)
{
	int i;
	mali_error mali_err = MALI_ERROR_NONE;
#ifdef CONFIG_KDS
	int err;
#endif				/* CONFIG_KDS */

	KBASE_DEBUG_ASSERT(kctx);

	kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", 0, 1);
	if (NULL == kctx->jctx.job_done_wq) {
		mali_err = MALI_ERROR_OUT_OF_MEMORY;
		goto out1;
	}

	for (i = 0; i < BASE_JD_ATOM_COUNT; i++) {
		init_waitqueue_head(&kctx->jctx.atoms[i].completed);

		INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]);
		INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]);

		/* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */
		kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID;
		kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED;
	}

	mutex_init(&kctx->jctx.lock);

	init_waitqueue_head(&kctx->jctx.zero_jobs_wait);

	spin_lock_init(&kctx->jctx.tb_lock);

#ifdef CONFIG_KDS
	err = kds_callback_init(&kctx->jctx.kds_cb, 0, kds_dep_clear);
	if (0 != err) {
		mali_err = MALI_ERROR_FUNCTION_FAILED;
		goto out2;
	}
#endif				/* CONFIG_KDS */

	kctx->jctx.job_nr = 0;

	return MALI_ERROR_NONE;

#ifdef CONFIG_KDS
 out2:
	destroy_workqueue(kctx->jctx.job_done_wq);
#endif				/* CONFIG_KDS */
 out1:
	return mali_err;
}

KBASE_EXPORT_TEST_API(kbase_jd_init)

void kbase_jd_exit(kbase_context *kctx)
{
	KBASE_DEBUG_ASSERT(kctx);

#ifdef CONFIG_KDS
	kds_callback_term(&kctx->jctx.kds_cb);
#endif				/* CONFIG_KDS */
	/* Work queue is emptied by this */
	destroy_workqueue(kctx->jctx.job_done_wq);
}

KBASE_EXPORT_TEST_API(kbase_jd_exit)
