drivers/gpu/drm/img-rogue/rgxray.c

/*************************************************************************/ /*!
@File
@Title          RGX ray tracing routines
@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
@Description    RGX ray tracing routines
@License        Dual MIT/GPLv2

The contents of this file are subject to the MIT license as set out below.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 2 ("GPL") in which case the provisions
of GPL are applicable instead of those above.

If you wish to allow use of your version of this file only under the terms of
GPL, and not to allow others to use your version of this file under the terms
of the MIT license, indicate your decision by deleting the provisions above
and replace them with the notice and other provisions required by GPL as set
out in the file called "GPL-COPYING" included in this distribution. If you do
not delete the provisions above, a recipient may use your version of this file
under the terms of either the MIT license or GPL.

This License is also included in this distribution in the file called
"MIT-COPYING".

EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ /**************************************************************************/
/* for the offsetof macro */
#include <stddef.h>
#if defined(INTEGRITY_OS)
#include <string.h>
#endif

#include "pdump_km.h"
#include "pvr_debug.h"
#include "rgxutils.h"
#include "rgxfwutils.h"
#include "rgxray.h"
#include "rgxmem.h"
#include "allocmem.h"
#include "devicemem.h"
#include "devicemem_pdump.h"
#include "devicemem_server.h"
#include "osfunc.h"
#include "pvrsrv.h"
#include "rgxccb.h"
#include "rgxhwperf.h"
#include "rgxtimerquery.h"
#include "htbuffer.h"

#include "rgxdefs_km.h"
#include "rgx_fwif_km.h"
#include "physmem.h"
#include "sync_server.h"
#include "sync_internal.h"
#include "sync.h"
#include "process_stats.h"


/*
 * FIXME: Defs copied from "rgxrpmdefs.h"
 */

typedef struct _RGX_RPM_DATA_RTU_FREE_PAGE_LIST {
     IMG_UINT32 u32_0; 
} RGX_RPM_DATA_RTU_FREE_PAGE_LIST;

/*
Page table index.
                                                        The field is a pointer to a free page 
*/
#define RGX_RPM_DATA_RTU_FREE_PAGE_LIST_PTI_WOFF          (0U)
#define RGX_RPM_DATA_RTU_FREE_PAGE_LIST_PTI_SHIFT         (0U)
#define RGX_RPM_DATA_RTU_FREE_PAGE_LIST_PTI_CLRMSK        (0XFFC00000U)
#define RGX_RPM_DATA_RTU_FREE_PAGE_LIST_SET_PTI(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_RPM_DATA_RTU_FREE_PAGE_LIST_PTI_CLRMSK ) | (((_x_) & (0x003fffff))  <<  0)))
#define RGX_RPM_DATA_RTU_FREE_PAGE_LIST_GET_PTI(_ft_)     (((_ft_).u32_0  >>  (0)) & 0x003fffff)

typedef struct _RGX_RPM_DATA_RTU_PAGE_TABLE {
     IMG_UINT32 u32_0; 
} RGX_RPM_DATA_RTU_PAGE_TABLE;

/*
 Page Table State
                                                        <br> 00: Empty Block
                                                        <br> 01: Full Block
                                                        <br> 10: Fragmented Block: Partially full page 
*/
#define RGX_RPM_DATA_RTU_PAGE_TABLE_PTS_WOFF              (0U)
#define RGX_RPM_DATA_RTU_PAGE_TABLE_PTS_SHIFT             (30U)
#define RGX_RPM_DATA_RTU_PAGE_TABLE_PTS_CLRMSK            (0X3FFFFFFFU)
#define RGX_RPM_DATA_RTU_PAGE_TABLE_SET_PTS(_ft_,_x_)     ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_RPM_DATA_RTU_PAGE_TABLE_PTS_CLRMSK ) | (((_x_) & (0x00000003))  <<  30)))
#define RGX_RPM_DATA_RTU_PAGE_TABLE_GET_PTS(_ft_)         (((_ft_).u32_0  >>  (30)) & 0x00000003)
/*
 Primitives in Page.
                                                        Number of unique primitives stored in this page.
                                                        The memory manager will re-use this page when the RCNT drops to zero.  
*/
#define RGX_RPM_DATA_RTU_PAGE_TABLE_RCNT_WOFF             (0U)
#define RGX_RPM_DATA_RTU_PAGE_TABLE_RCNT_SHIFT            (22U)
#define RGX_RPM_DATA_RTU_PAGE_TABLE_RCNT_CLRMSK           (0XC03FFFFFU)
#define RGX_RPM_DATA_RTU_PAGE_TABLE_SET_RCNT(_ft_,_x_)    ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_RPM_DATA_RTU_PAGE_TABLE_RCNT_CLRMSK ) | (((_x_) & (0x000000ff))  <<  22)))
#define RGX_RPM_DATA_RTU_PAGE_TABLE_GET_RCNT(_ft_)        (((_ft_).u32_0  >>  (22)) & 0x000000ff)
/*
Next page table index.
                                                        The field is a pointer to the next page for this primitive.  
*/
#define RGX_RPM_DATA_RTU_PAGE_TABLE_NPTI_WOFF             (0U)
#define RGX_RPM_DATA_RTU_PAGE_TABLE_NPTI_SHIFT            (0U)
#define RGX_RPM_DATA_RTU_PAGE_TABLE_NPTI_CLRMSK           (0XFFC00000U)
#define RGX_RPM_DATA_RTU_PAGE_TABLE_SET_NPTI(_ft_,_x_)    ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_RPM_DATA_RTU_PAGE_TABLE_NPTI_CLRMSK ) | (((_x_) & (0x003fffff))  <<  0)))
#define RGX_RPM_DATA_RTU_PAGE_TABLE_GET_NPTI(_ft_)        (((_ft_).u32_0  >>  (0)) & 0x003fffff)


#define RGX_CR_RPM_PAGE_TABLE_BASE_VALUE_ALIGNSHIFT		(2U)
#define RGX_CR_RPM_SHF_FPL_BASE_ALIGNSHIFT				(2U)


typedef struct {
	DEVMEM_MEMDESC				*psContextStateMemDesc;
	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
	IMG_UINT32					ui32Priority;
#if 0	
	/* FIXME - multiple frame contexts? */
	RGX_RPM_FREELIST				*psSHFFreeList;
	RGX_RPM_FREELIST				*psSHGFreeList;
#endif
} RGX_SERVER_RAY_SH_DATA;


typedef enum {
	NODE_EMPTY = 0,
	NODE_SCENE_HIERARCHY,
	NODE_RPM_PAGE_TABLE,
	NODE_RPM_FREE_PAGE_LIST
} RGX_DEVMEM_NODE_TYPE;

typedef struct _RGX_DEVMEM_NODE_ {
	RGX_DEVMEM_NODE_TYPE	eNodeType;			/*!< Alloc type */
	PMR						*psPMR; 			/*!< Scene hierarchy/page table/free page list phys pages */
	DEVMEMINT_HEAP			*psDevMemHeap;		/*!< Heap where the virtual mapping is made */
	IMG_DEV_VIRTADDR		sAddr;				/*!< GPU virtual address where the phys pages are mapped into */
	IMG_UINT32				ui32NumPhysPages;	/*!< Number of physical pages mapped in for this node */
	IMG_UINT32				ui32StartOfMappingIndex;	/*!< Start of mapping index (i.e. OS page offset from virtual base) */
	IMG_BOOL				bInternal;
} RGX_DEVMEM_NODE;

typedef struct _RGX_RPM_DEVMEM_DESC_ {
	DLLIST_NODE			sMemoryDescBlock;		/*!< the hierarchy scene memory block  */
	RGX_RPM_FREELIST	*psFreeList;			/*!< Free list this allocation is associated with */
	IMG_UINT32			ui32NumPages;			/*!< Number of RPM pages added */
	RGX_DEVMEM_NODE		sSceneHierarchyNode;	/*!< scene hierarchy block descriptor */
	RGX_DEVMEM_NODE		sRPMPageListNode;		/*!< RPM page list block descriptor */
	RGX_DEVMEM_NODE		sRPMFreeListNode;		/*!< RPM free list block descriptor */
} RGX_RPM_DEVMEM_DESC;

typedef struct _DEVMEM_RPM_FREELIST_LOOKUP_
{
	IMG_UINT32 ui32FreeListID;
	RGX_RPM_FREELIST *psFreeList;
} DEVMEM_RPM_FREELIST_LOOKUP;

typedef struct {
	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
	IMG_UINT32					ui32Priority;
	RGX_CLIENT_CCB *psFCClientCCB[DPX_MAX_RAY_CONTEXTS];
	DEVMEM_MEMDESC *psFCClientCCBMemDesc[DPX_MAX_RAY_CONTEXTS];
	DEVMEM_MEMDESC *psFCClientCCBCtrlMemDesc[DPX_MAX_RAY_CONTEXTS];
} RGX_SERVER_RAY_RS_DATA;


struct _RGX_SERVER_RAY_CONTEXT_ {
	PVRSRV_DEVICE_NODE			*psDeviceNode;
	DEVMEM_MEMDESC				*psFWRayContextMemDesc;
	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
	RGX_SERVER_RAY_SH_DATA		sSHData;
	RGX_SERVER_RAY_RS_DATA		sRSData;
	IMG_UINT32					ui32CleanupStatus;
#define RAY_CLEANUP_SH_COMPLETE		(1 << 0)
#define RAY_CLEANUP_RS_COMPLETE		(1 << 1)
	PVRSRV_CLIENT_SYNC_PRIM		*psCleanupSync;
	DLLIST_NODE					sListNode;
	SYNC_ADDR_LIST				sSyncAddrListFence;
	SYNC_ADDR_LIST				sSyncAddrListUpdate;
	ATOMIC_T					hJobId;
};


#if 0
static
#ifdef __GNUC__
	__attribute__((noreturn))
#endif
void sleep_for_ever(void)
{
#if defined(__KLOCWORK__) // klocworks would report an infinite loop because of while(1).
	PVR_ASSERT(0); 
#else
	while(1)
	{
		OSSleepms(~0); // sleep the maximum amount of time possible
	}
#endif
}
#endif

static
PVRSRV_ERROR _RGXCreateRPMSparsePMR(CONNECTION_DATA *psConnection,
									PVRSRV_DEVICE_NODE	 *psDeviceNode,
									RGX_DEVMEM_NODE_TYPE eBlockType,
									IMG_UINT32		ui32NumPages,
									IMG_UINT32		uiLog2DopplerPageSize,
									PMR				**ppsPMR);

static PVRSRV_ERROR _RGXMapRPMPBBlock(RGX_DEVMEM_NODE	*psDevMemNode,
					RGX_RPM_FREELIST *psFreeList,
					RGX_DEVMEM_NODE_TYPE eBlockType,
					DEVMEMINT_HEAP *psDevmemHeap,
					IMG_UINT32 ui32NumPages,
					IMG_DEV_VIRTADDR sDevVAddrBase);

static
PVRSRV_ERROR _RGXUnmapRPMPBBlock(RGX_DEVMEM_NODE	*psDevMemNode,
					RGX_RPM_FREELIST *psFreeList,
					IMG_DEV_VIRTADDR sDevVAddrBase);

static
PVRSRV_ERROR _CreateSHContext(CONNECTION_DATA *psConnection,
							  PVRSRV_DEVICE_NODE *psDeviceNode,
							  DEVMEM_MEMDESC *psAllocatedMemDesc,
							  IMG_UINT32 ui32AllocatedOffset,
							  DEVMEM_MEMDESC *psFWMemContextMemDesc,
							  IMG_DEV_VIRTADDR sVRMCallStackAddr,
							  IMG_UINT32 ui32Priority,
							  RGX_COMMON_CONTEXT_INFO *psInfo,
							  RGX_SERVER_RAY_SH_DATA *psSHData)
{
	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
	RGXFWIF_VRDMCTX_STATE *psContextState;
	PVRSRV_ERROR eError;
	/*
		Allocate device memory for the firmware GPU context suspend state.
		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
	*/
	PDUMPCOMMENT("Allocate RGX firmware SHG context suspend state");

	eError = DevmemFwAllocate(psDevInfo,
							  sizeof(RGXFWIF_VRDMCTX_STATE),
							  RGX_FWCOMCTX_ALLOCFLAGS,
							  "FwRaySHGContextSuspendState",
							  &psSHData->psContextStateMemDesc);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRayContextKM: Failed to allocate firmware GPU context suspend state (%u)",
				eError));
		goto fail_shcontextsuspendalloc;
	}

	eError = DevmemAcquireCpuVirtAddr(psSHData->psContextStateMemDesc,
                                      (void **)&psContextState);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRayContextKM: Failed to map firmware render context state (%u)",
				eError));
		goto fail_suspendcpuvirtacquire;
	}
	psContextState->uVRDMReg_VRM_CALL_STACK_POINTER = sVRMCallStackAddr.uiAddr;
	DevmemReleaseCpuVirtAddr(psSHData->psContextStateMemDesc);

	eError = FWCommonContextAllocate(psConnection,
									 psDeviceNode,
									 REQ_TYPE_SH,
									 RGXFWIF_DM_SHG,
									 psAllocatedMemDesc,
									 ui32AllocatedOffset,
									 psFWMemContextMemDesc,
									 psSHData->psContextStateMemDesc,
									 RGX_RTU_CCB_SIZE_LOG2,
									 ui32Priority,
									 psInfo,
									 &psSHData->psServerCommonContext);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRayContextKM: Failed to init TA fw common context (%u)",
				eError));
		goto fail_shcommoncontext;
	}
	
	/*
	 * Dump the FW SH context suspend state buffer
	 */
	PDUMPCOMMENT("Dump the SH context suspend state buffer");
	DevmemPDumpLoadMem(psSHData->psContextStateMemDesc,
					   0,
					   sizeof(RGXFWIF_VRDMCTX_STATE),
					   PDUMP_FLAGS_CONTINUOUS);

	psSHData->ui32Priority = ui32Priority;
	return PVRSRV_OK;

fail_shcommoncontext:
fail_suspendcpuvirtacquire:
	DevmemFwFree(psDevInfo, psSHData->psContextStateMemDesc);
fail_shcontextsuspendalloc:
	PVR_ASSERT(eError != PVRSRV_OK);

	return eError;
}

static
PVRSRV_ERROR _CreateRSContext(CONNECTION_DATA *psConnection,
							  PVRSRV_DEVICE_NODE *psDeviceNode,
							  DEVMEM_MEMDESC *psAllocatedMemDesc,
							  IMG_UINT32 ui32AllocatedOffset,
							  DEVMEM_MEMDESC *psFWMemContextMemDesc,
							  IMG_UINT32 ui32Priority,
							  RGX_COMMON_CONTEXT_INFO *psInfo,
							  RGX_SERVER_RAY_RS_DATA *psRSData)
{
	PVRSRV_ERROR eError;

	eError = FWCommonContextAllocate(psConnection,
									 psDeviceNode,
									 REQ_TYPE_RS,
									 RGXFWIF_DM_RTU,
									 psAllocatedMemDesc,
									 ui32AllocatedOffset,
									 psFWMemContextMemDesc,
                                     NULL,
									 RGX_RTU_CCB_SIZE_LOG2,
									 ui32Priority,
									 psInfo,
									 &psRSData->psServerCommonContext);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRayContextKM: Failed to init 3D fw common context (%u)",
				eError));
		goto fail_rscommoncontext;
	}

	psRSData->ui32Priority = ui32Priority;
	return PVRSRV_OK;

fail_rscommoncontext:
	PVR_ASSERT(eError != PVRSRV_OK);

	return eError;
}


/*
	Static functions used by ray context code
*/

static
PVRSRV_ERROR _DestroySHContext(RGX_SERVER_RAY_SH_DATA *psSHData,
							   PVRSRV_DEVICE_NODE *psDeviceNode,
							   PVRSRV_CLIENT_SYNC_PRIM *psCleanupSync)
{
	PVRSRV_ERROR eError;

	/* Check if the FW has finished with this resource ... */
	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
											  psSHData->psServerCommonContext,
											  psCleanupSync,
											  RGXFWIF_DM_SHG,
											  PDUMP_FLAGS_NONE);
	if (eError == PVRSRV_ERROR_RETRY)
	{
		return eError;
	}
	else if (eError != PVRSRV_OK)
	{
		PVR_LOG(("%s: Unexpected error from RGXFWRequestCommonContextCleanUp (%s)",
				__FUNCTION__,
				PVRSRVGetErrorStringKM(eError)));
		return eError;
	}

	/* ... it has so we can free its resources */
	FWCommonContextFree(psSHData->psServerCommonContext);
	DevmemFwFree(psDeviceNode->pvDevice, psSHData->psContextStateMemDesc);
	psSHData->psContextStateMemDesc = NULL;
	psSHData->psServerCommonContext = NULL;
	return PVRSRV_OK;
}

static
PVRSRV_ERROR _DestroyRSContext(RGX_SERVER_RAY_RS_DATA *psRSData,
							   PVRSRV_DEVICE_NODE *psDeviceNode,
							   PVRSRV_CLIENT_SYNC_PRIM *psCleanupSync)
{
	PVRSRV_ERROR eError;

	/* Check if the FW has finished with this resource ... */
	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
											  psRSData->psServerCommonContext,
											  psCleanupSync,
											  RGXFWIF_DM_RTU,
											  PDUMP_FLAGS_NONE);
	if (eError == PVRSRV_ERROR_RETRY)
	{
		return eError;
	}
	else if (eError != PVRSRV_OK)
	{
		PVR_LOG(("%s: Unexpected error from RGXFWRequestCommonContextCleanUp (%s)",
				 __FUNCTION__,
				 PVRSRVGetErrorStringKM(eError)));
		return eError;
	}

	/* ... it has so we can free its resources */


	FWCommonContextFree(psRSData->psServerCommonContext);
	psRSData->psServerCommonContext = NULL;
	return PVRSRV_OK;
}


/*
 * RPM driver management rev 2
 * 
 * The RPM freelists are opaque to the client driver. Scene Hierarchy pages
 * are managed in Blocks (analogous to PB blocks) which are alloc'd in KM
 * and mapped into the client MMU context.
 * 
 * Page tables are set up for each existing Scene Memory Block.
 * 
 * Freelist entries are also updated according to the list of Scene Memory Blocks.
 * 
 * NOTES:
 * 
 * (1) Scene Hierarchy shrink is not expected to be used.
 * (2) The RPM FreeLists are Circular buffers and must be contiguous in virtual space
 * (3) Each PMR is created with no phys backing pages. Pages are mapped in on-demand
 * via RGXGrowRPMFreeList.
 * 
 */
#if defined(DEBUG)
static PVRSRV_ERROR _ReadRPMFreePageList(PMR		 *psPMR,
										 IMG_DEVMEM_OFFSET_T uiLogicalOffset,
										 IMG_UINT32  ui32PageCount)
{
	PVRSRV_ERROR	eError;
	IMG_UINT32		uiIdx, j;
	size_t			uNumBytesCopied;
	RGX_RPM_DATA_RTU_FREE_PAGE_LIST		*psFreeListBuffer;
	IMG_UINT32		ui32PTI[4];

	/* Allocate scratch area for setting up Page table indices */
	psFreeListBuffer = OSAllocMem(ui32PageCount * sizeof(RGX_RPM_DATA_RTU_FREE_PAGE_LIST));
    if (psFreeListBuffer == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR, "_WriteRPMPageList: failed to allocate scratch page table"));
		return PVRSRV_ERROR_OUT_OF_MEMORY;
	}
	
	/* Read scratch buffer from PMR (FPL entries must be contiguous) */
	eError = PMR_ReadBytes(psPMR,
				 uiLogicalOffset,
				 (IMG_UINT8 *) psFreeListBuffer,
				 ui32PageCount * sizeof(RGX_RPM_DATA_RTU_FREE_PAGE_LIST),
				 &uNumBytesCopied);

	if (eError == PVRSRV_OK)
	{
		for (uiIdx = 0; uiIdx < ui32PageCount; uiIdx +=4)
		{
			for (j=0; j<4; j++)
			{
				ui32PTI[j] = RGX_RPM_DATA_RTU_FREE_PAGE_LIST_GET_PTI(psFreeListBuffer[uiIdx + j]);
			}
			PVR_DPF((PVR_DBG_MESSAGE, "%4d:  %7d %7d %7d %7d", uiIdx,
					ui32PTI[0], ui32PTI[1], ui32PTI[2], ui32PTI[3]));
		}
	}

	/* Free scratch buffer */
	OSFreeMem(psFreeListBuffer);

	return eError;
}

static IMG_BOOL RGXDumpRPMFreeListPageList(RGX_RPM_FREELIST *psFreeList)
{
	PVR_LOG(("RPM Freelist FWAddr 0x%08x, ID = %d, CheckSum 0x%016llx",
				psFreeList->sFreeListFWDevVAddr.ui32Addr,
				psFreeList->ui32FreelistID,
				psFreeList->ui64FreelistChecksum));

	/* Dump FreeList page list */
	_ReadRPMFreePageList(psFreeList->psFreeListPMR, 0, psFreeList->ui32CurrentFLPages);

	return IMG_TRUE;
}
#endif

static PVRSRV_ERROR _UpdateFwRPMFreelistSize(RGX_RPM_FREELIST *psFreeList,
											 IMG_BOOL bGrow,
											 IMG_BOOL bRestartRPM,
											 IMG_UINT32 ui32DeltaSize)
{
	PVRSRV_ERROR			eError;
	RGXFWIF_KCCB_CMD		sGPCCBCmd;

	if(!bGrow)
	{
		PVR_DPF((PVR_DBG_ERROR, "_UpdateFwRPMFreelistSize: RPM freelist shrink not supported."));
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	/* send feedback */
	sGPCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_DOPPLER_MEMORY_GROW;
	sGPCCBCmd.uCmdData.sFreeListGSData.sFreeListFWDevVAddr.ui32Addr = psFreeList->sFreeListFWDevVAddr.ui32Addr;
	sGPCCBCmd.uCmdData.sFreeListGSData.ui32DeltaSize = ui32DeltaSize;
	sGPCCBCmd.uCmdData.sFreeListGSData.ui32NewSize = 
		((bRestartRPM) ? RGX_FREELIST_GSDATA_RPM_RESTART_EN : 0) |
		psFreeList->ui32CurrentFLPages;

	PVR_DPF((PVR_DBG_MESSAGE, "Send FW update: RPM freelist [FWAddr=0x%08x] has 0x%08x pages",
								psFreeList->sFreeListFWDevVAddr.ui32Addr,
								psFreeList->ui32CurrentFLPages));

	/* Submit command to the firmware.  */
	LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
	{
		eError = RGXScheduleCommand(psFreeList->psDevInfo,
									RGXFWIF_DM_GP,
									&sGPCCBCmd,
									sizeof(sGPCCBCmd),
									0,
									PDUMP_FLAGS_CONTINUOUS);
		if (eError != PVRSRV_ERROR_RETRY)
		{
			break;
		}
		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
	} END_LOOP_UNTIL_TIMEOUT();

	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "_UpdateFwRPMFreelistSize: failed to update FW freelist size. (error = %u)", eError));
		return eError;
	}

	return PVRSRV_OK;
}

#if 0
static void _CheckRPMFreelist(RGX_RPM_FREELIST *psFreeList,
                   	   	   	   IMG_UINT32 ui32NumOfPagesToCheck,
                   	   	   	   IMG_UINT64 ui64ExpectedCheckSum,
                   	   	   	   IMG_UINT64 *pui64CalculatedCheckSum)
{
#if defined(NO_HARDWARE)
	/* No checksum needed as we have all information in the pdumps */
	PVR_UNREFERENCED_PARAMETER(psFreeList);
	PVR_UNREFERENCED_PARAMETER(ui32NumOfPagesToCheck);
	PVR_UNREFERENCED_PARAMETER(ui64ExpectedCheckSum);
	*pui64CalculatedCheckSum = 0;
#else
	PVRSRV_ERROR eError;
	size_t uiNumBytes;
    IMG_UINT8* pui8Buffer;
    IMG_UINT32* pui32Buffer;
    IMG_UINT32 ui32CheckSumAdd = 0;
    IMG_UINT32 ui32CheckSumXor = 0;
    IMG_UINT32 ui32Entry;
    IMG_UINT32 ui32Entry2;
    IMG_BOOL  bFreelistBad = IMG_FALSE;

	*pui64CalculatedCheckSum = 0;

	/* Allocate Buffer of the size of the freelist */
	pui8Buffer = OSAllocMem(psFreeList->ui32CurrentFLPages * sizeof(IMG_UINT32));
    if (pui8Buffer == NULL)
    {
		PVR_LOG(("_CheckRPMFreelist: Failed to allocate buffer to check freelist %p!", psFreeList));
		sleep_for_ever();
		//PVR_ASSERT(0);
        return;
    }

    /* Copy freelist content into Buffer */
    eError = PMR_ReadBytes(psFreeList->psFreeListPMR,
    				psFreeList->uiFreeListPMROffset + (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32),
    				pui8Buffer,
    				psFreeList->ui32CurrentFLPages * sizeof(IMG_UINT32),
            		&uiNumBytes);
    if (eError != PVRSRV_OK)
    {
		OSFreeMem(pui8Buffer);
		PVR_LOG(("_CheckRPMFreelist: Failed to get freelist data for RPM freelist %p!", psFreeList));
		sleep_for_ever();
		//PVR_ASSERT(0);
        return;
    }

    PVR_ASSERT(uiNumBytes == psFreeList->ui32CurrentFLPages * sizeof(IMG_UINT32));
    PVR_ASSERT(ui32NumOfPagesToCheck <= psFreeList->ui32CurrentFLPages);

    /* Generate checksum */
    pui32Buffer = (IMG_UINT32 *)pui8Buffer;
    for(ui32Entry = 0; ui32Entry < ui32NumOfPagesToCheck; ui32Entry++)
    {
    	ui32CheckSumAdd += pui32Buffer[ui32Entry];
    	ui32CheckSumXor ^= pui32Buffer[ui32Entry];

    	/* Check for double entries */
    	for (ui32Entry2 = 0; ui32Entry2 < ui32NumOfPagesToCheck; ui32Entry2++)
    	{
			if ((ui32Entry != ui32Entry2) &&
				(pui32Buffer[ui32Entry] == pui32Buffer[ui32Entry2]))
			{
				PVR_LOG(("_CheckRPMFreelist: RPM Freelist consistency failure: FW addr: 0x%08X, Double entry found 0x%08x on idx: %d and %d of %d",
											psFreeList->sFreeListFWDevVAddr.ui32Addr,
											pui32Buffer[ui32Entry2],
											ui32Entry,
											ui32Entry2,
											psFreeList->ui32CurrentFLPages));
				bFreelistBad = IMG_TRUE;
			}
    	}
    }

    OSFreeMem(pui8Buffer);

	/* Check the calculated checksum against the expected checksum... */
	*pui64CalculatedCheckSum = ((IMG_UINT64)ui32CheckSumXor << 32) | ui32CheckSumAdd;

	if (ui64ExpectedCheckSum != 0  &&  ui64ExpectedCheckSum != *pui64CalculatedCheckSum)
	{
		PVR_LOG(("_CheckRPMFreelist: Checksum mismatch for RPM freelist %p!  Expected 0x%016llx calculated 0x%016llx",
		        psFreeList, ui64ExpectedCheckSum, *pui64CalculatedCheckSum));
		bFreelistBad = IMG_TRUE;
	}
    
    if (bFreelistBad)
    {
		PVR_LOG(("_CheckRPMFreelist: Sleeping for ever!"));
		sleep_for_ever();
//		PVR_ASSERT(!bFreelistBad);
	}
#endif
}
#endif

static PVRSRV_ERROR _WriteRPMFreePageList(PMR		 *psPMR,
										  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
										  IMG_UINT32  ui32NextPageIndex,
										  IMG_UINT32  ui32PageCount)
{
	PVRSRV_ERROR	eError;
	IMG_UINT32		uiIdx;
	size_t		uNumBytesCopied;
	RGX_RPM_DATA_RTU_FREE_PAGE_LIST		*psFreeListBuffer;

	/* Allocate scratch area for setting up Page table indices */
	psFreeListBuffer = OSAllocMem(ui32PageCount * sizeof(RGX_RPM_DATA_RTU_FREE_PAGE_LIST));
    if (psFreeListBuffer == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR, "_WriteRPMPageList: failed to allocate scratch page table"));
		return PVRSRV_ERROR_OUT_OF_MEMORY;
	}
	
	for (uiIdx = 0; uiIdx < ui32PageCount; uiIdx ++, ui32NextPageIndex ++)
	{
		psFreeListBuffer[uiIdx].u32_0 = 0;
		RGX_RPM_DATA_RTU_FREE_PAGE_LIST_SET_PTI(psFreeListBuffer[uiIdx], ui32NextPageIndex);
	}
	
	/* Copy scratch buffer to PMR */
	eError = PMR_WriteBytes(psPMR,
				 uiLogicalOffset,
				 (IMG_UINT8 *) psFreeListBuffer,
				 ui32PageCount * sizeof(RGX_RPM_DATA_RTU_FREE_PAGE_LIST),
				 &uNumBytesCopied);
	
	/* Free scratch buffer */
	OSFreeMem(psFreeListBuffer);

#if defined(PDUMP)
	/* Pdump the Page tables */
	PDUMPCOMMENT("Dump %u RPM free page list entries.", ui32PageCount);
	PMRPDumpLoadMem(psPMR,
					uiLogicalOffset,
					ui32PageCount * sizeof(RGX_RPM_DATA_RTU_FREE_PAGE_LIST),
					PDUMP_FLAGS_CONTINUOUS,
					IMG_FALSE);
#endif
	return eError;
}


static RGX_RPM_FREELIST* FindRPMFreeList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FreelistID)
{
	DLLIST_NODE *psNode, *psNext;
	RGX_RPM_FREELIST *psFreeList = NULL;

	OSLockAcquire(psDevInfo->hLockRPMFreeList);
	dllist_foreach_node(&psDevInfo->sRPMFreeListHead, psNode, psNext)
	{
		RGX_RPM_FREELIST *psThisFreeList = IMG_CONTAINER_OF(psNode, RGX_RPM_FREELIST, sNode);

		if (psThisFreeList->ui32FreelistID == ui32FreelistID)
		{
			psFreeList = psThisFreeList;
			break;
		}
	}
	OSLockRelease(psDevInfo->hLockRPMFreeList);
	
	return psFreeList;
}

void RGXProcessRequestRPMGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
							  IMG_UINT32 ui32FreelistID)
{
	RGX_RPM_FREELIST *psFreeList = NULL;
	RGXFWIF_KCCB_CMD sVRDMCCBCmd;
	IMG_UINT32 ui32GrowValue;
	PVRSRV_ERROR eError;
	IMG_BOOL bRestartRPM = IMG_TRUE; /* FIXME */

	PVR_ASSERT(psDevInfo);

	/* find the freelist with the corresponding ID */
	psFreeList = FindRPMFreeList(psDevInfo, ui32FreelistID);

	if (psFreeList)
	{
		/* Try to grow the freelist */
		eError = RGXGrowRPMFreeList(psFreeList,
									psFreeList->ui32GrowFLPages,
									&psFreeList->sMemoryBlockHead);
		if (eError == PVRSRV_OK)
		{
			/* Grow successful, return size of grow size */
			ui32GrowValue = psFreeList->ui32GrowFLPages;

			psFreeList->ui32NumGrowReqByFW++;

 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
			/* Update Stats */
			PVRSRVStatsUpdateFreelistStats(0,
	                               1, /* Add 1 to the appropriate counter (Requests by FW) */
	                               psFreeList->ui32InitFLPages,
	                               psFreeList->ui32NumHighPages,
	                               psFreeList->ownerPid);

 #endif

		}
		else
		{
			/* Grow failed */
			ui32GrowValue = 0;
			PVR_DPF((PVR_DBG_ERROR,"Grow for FreeList %p [ID %d] failed (error %u)",
									psFreeList,
									psFreeList->ui32FreelistID,
									eError));
		}

		/* send feedback */
		sVRDMCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_DOPPLER_MEMORY_GROW;
		sVRDMCCBCmd.uCmdData.sFreeListGSData.sFreeListFWDevVAddr.ui32Addr = psFreeList->sFreeListFWDevVAddr.ui32Addr;
		sVRDMCCBCmd.uCmdData.sFreeListGSData.ui32DeltaSize = ui32GrowValue;
		sVRDMCCBCmd.uCmdData.sFreeListGSData.ui32NewSize = 
			((bRestartRPM) ? RGX_FREELIST_GSDATA_RPM_RESTART_EN : 0) |
			(psFreeList->ui32CurrentFLPages);

		PVR_DPF((PVR_DBG_ERROR,"Send feedback to RPM after grow on freelist [ID %d]", ui32FreelistID));
		LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
		{
			eError = RGXScheduleCommand(psDevInfo,
										RGXFWIF_DM_SHG,
										&sVRDMCCBCmd,
										sizeof(sVRDMCCBCmd),
										0,
										PDUMP_FLAGS_NONE);
			if (eError != PVRSRV_ERROR_RETRY)
			{
				break;
			}
			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
		} END_LOOP_UNTIL_TIMEOUT();
		/* Kernel CCB should never fill up, as the FW is processing them right away  */

		PVR_ASSERT(eError == PVRSRV_OK);
	}
	else
	{
		/* Should never happen */
		PVR_DPF((PVR_DBG_ERROR,"FreeList Lookup for FreeList ID 0x%08x failed (Populate)", ui32FreelistID));
		PVR_ASSERT(IMG_FALSE);
	}
}


/*!
 * RGXGrowRPMFreeList
 *
 * Allocate and map physical backing pages for RPM buffers
 * 
 * @param	ppsRPMDevMemDesc - RPM buffer descriptor representing new Scene memory block
 * 								and its associated RPM page table and free page list entries
 * @param	psRPMContext - RPM context
 * @param	psFreeList - RPM freelist descriptor
 * @param	ui32RequestNumPages - number of RPM pages to add to Doppler scene hierarchy
 * @param	pListHeader - linked list of RGX_RPM_DEVMEM_DESC blocks
 * 
 */
PVRSRV_ERROR RGXGrowRPMFreeList(RGX_RPM_FREELIST *psFreeList,
								IMG_UINT32 ui32RequestNumPages,
								PDLLIST_NODE pListHeader)
{
	PVRSRV_ERROR			eError;
	RGX_SERVER_RPM_CONTEXT	*psRPMContext = psFreeList->psParentCtx;
	RGX_RPM_DEVMEM_DESC		*psRPMDevMemDesc;
	IMG_DEVMEM_OFFSET_T		uiPMROffset;
	IMG_UINT32				ui32NextPageIndex;

	/* Are we allowed to grow ? */
	if (ui32RequestNumPages > psFreeList->psParentCtx->ui32UnallocatedPages)
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXGrowRPMFreeList: Scene Hierarchy buffer exceeded (0x%x pages required, 0x%x pages available).",
				ui32RequestNumPages, psFreeList->psParentCtx->ui32UnallocatedPages));
		return PVRSRV_ERROR_RPM_PBSIZE_ALREADY_MAX;
	}

	/* Allocate descriptor */
	psRPMDevMemDesc = OSAllocZMem(sizeof(*psRPMDevMemDesc));
    if (psRPMDevMemDesc == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXGrowRPMFreeList: failed to allocate host data structure"));
		return PVRSRV_ERROR_OUT_OF_MEMORY;
	}

	/*
	 * Lock protects simultaneous manipulation of:
	 * - the memory block list
	 * - the freelist's ui32CurrentFLPages
	 * - the context's ui32UnallocatedPages
	 */
	OSLockAcquire(psFreeList->psDevInfo->hLockRPMFreeList);
	OSLockAcquire(psFreeList->psDevInfo->hLockRPMContext);

	/* Update the sparse PMRs */
	psRPMDevMemDesc->psFreeList = psFreeList;
	psRPMDevMemDesc->ui32NumPages = ui32RequestNumPages;
	psRPMDevMemDesc->sSceneHierarchyNode.psPMR = psRPMContext->psSceneHierarchyPMR;
	psRPMDevMemDesc->sRPMPageListNode.psPMR = psRPMContext->psRPMPageTablePMR;
	psRPMDevMemDesc->sRPMFreeListNode.psPMR = psFreeList->psFreeListPMR;


	PVR_DPF((PVR_DBG_MESSAGE, "RGXGrowRPMFreeList: mapping %d pages for Doppler scene memory to VA 0x%llx with heap ID %p",
			ui32RequestNumPages, psRPMContext->sSceneMemoryBaseAddr.uiAddr, psRPMContext->psSceneHeap));

	/* 
	 * 1. Doppler scene hierarchy
	 */
	PDUMPCOMMENT("Allocate %d pages with mapping index %d for Doppler scene memory.",
				 ui32RequestNumPages,
				 psRPMContext->ui32SceneMemorySparseMappingIndex);
	eError = _RGXMapRPMPBBlock(&psRPMDevMemDesc->sSceneHierarchyNode,
					psFreeList,
					NODE_SCENE_HIERARCHY,
					psRPMContext->psSceneHeap,
					ui32RequestNumPages,
					psRPMContext->sSceneMemoryBaseAddr);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXGrowRPMFreeList: Unable to map RPM scene hierarchy block (status %d)", eError));
		goto ErrorSceneBlock;
	}

	/* 
	 * 2. RPM page list
	 */
	if (ui32RequestNumPages > psRPMContext->ui32RPMEntriesInPage)
	{
		/* we need to map in phys pages for RPM page table */
		PDUMPCOMMENT("Allocate %d (%d requested) page table entries with mapping index %d for RPM page table.",
					 ui32RequestNumPages - psRPMContext->ui32RPMEntriesInPage,
					 ui32RequestNumPages,
					 psRPMContext->ui32RPMPageTableSparseMappingIndex);
		eError = _RGXMapRPMPBBlock(&psRPMDevMemDesc->sRPMPageListNode,
						psFreeList,
						NODE_RPM_PAGE_TABLE,
						psRPMContext->psRPMPageTableHeap,
						ui32RequestNumPages - psRPMContext->ui32RPMEntriesInPage,
						psRPMContext->sRPMPageTableBaseAddr);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR, "RGXGrowRPMFreeList: Unable to map RPM page table block (status %d)", eError));
			goto ErrorPageTableBlock;
		}
	}

	/*
	 * 3. Free page list (FPL)
	 */
	if (ui32RequestNumPages > psFreeList->ui32EntriesInPage)
	{
		/* we need to map in phys pages for RPM free page list */
		PDUMPCOMMENT("Allocate %d (%d requested) FPL entries with mapping index %d for RPM free page list.",
					 ui32RequestNumPages - psFreeList->ui32EntriesInPage,
					 ui32RequestNumPages,
					 psFreeList->ui32RPMFreeListSparseMappingIndex);
		eError = _RGXMapRPMPBBlock(&psRPMDevMemDesc->sRPMFreeListNode,
						psFreeList,
						NODE_RPM_FREE_PAGE_LIST,
						psRPMContext->psRPMPageTableHeap,
						ui32RequestNumPages - psFreeList->ui32EntriesInPage,
						psFreeList->sBaseDevVAddr);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR, "RGXGrowRPMFreeList: Unable to map RPM free page list (status %d)", eError));
			goto ErrorFreeListBlock;
		}
	}

	/*
	 * Update FPL entries
	 */

	/* Calculate doppler page index from base of Doppler heap */
	ui32NextPageIndex = (psRPMDevMemDesc->sSceneHierarchyNode.sAddr.uiAddr -
		psRPMContext->sDopplerHeapBaseAddr.uiAddr) >> psFreeList->uiLog2DopplerPageSize;

	/* Calculate write offset into FPL PMR assuming pages are mapped in order with no gaps */
	uiPMROffset = (size_t)psFreeList->ui32CurrentFLPages * sizeof(RGX_RPM_DATA_RTU_FREE_PAGE_LIST);

	eError = _WriteRPMFreePageList(psFreeList->psFreeListPMR, uiPMROffset, ui32NextPageIndex, ui32RequestNumPages);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXGrowRPMFreeList: error writing RPM free list entries (%d)", eError));
		goto ErrorFreeListWriteEntries;
	}

	{
		/* 
		 * Update the entries remaining in the last mapped RPM and FPL pages.
		 * 
		 * psRPMDevMemDesc->sRPMPageListNode.ui32NumPhysPages * 1024 entries are added (can be zero)
		 * ui32RequestNumPages entries are committed
		 * 
		 * The number of entries remaining should always be less than a full page.
		 */
		IMG_UINT32	ui32PTEntriesPerChunk = OSGetPageSize() / sizeof(RGX_RPM_DATA_RTU_FREE_PAGE_LIST);
		IMG_UINT32	ui32PTEntriesPerChunkClearMask = ~(ui32PTEntriesPerChunk - 1);

		psRPMContext->ui32RPMEntriesInPage = psRPMContext->ui32RPMEntriesInPage +
			(psRPMDevMemDesc->sRPMPageListNode.ui32NumPhysPages * ui32PTEntriesPerChunk) - ui32RequestNumPages;
		PVR_ASSERT((psRPMContext->ui32RPMEntriesInPage & ui32PTEntriesPerChunkClearMask) == 0);

		psFreeList->ui32EntriesInPage = psFreeList->ui32EntriesInPage +
			(psRPMDevMemDesc->sRPMFreeListNode.ui32NumPhysPages * ui32PTEntriesPerChunk) - ui32RequestNumPages;
		PVR_ASSERT((psFreeList->ui32EntriesInPage & ui32PTEntriesPerChunkClearMask) == 0);
	}

	/* Add node to link list */
	dllist_add_to_head(pListHeader, &psRPMDevMemDesc->sMemoryDescBlock);

	/* Update number of available pages */
	psFreeList->ui32CurrentFLPages += ui32RequestNumPages;
	psRPMContext->ui32UnallocatedPages -= ui32RequestNumPages;

#if defined(DEBUG)
	RGXDumpRPMFreeListPageList(psFreeList);
#endif

	OSLockRelease(psFreeList->psDevInfo->hLockRPMContext);
	OSLockRelease(psFreeList->psDevInfo->hLockRPMFreeList);

	PVR_DPF((PVR_DBG_MESSAGE,"RPM Freelist [%p, ID %d]: grow by %u pages (current pages %u/%u, unallocated pages %u)",
			psFreeList,
			psFreeList->ui32FreelistID,
			ui32RequestNumPages,
			psFreeList->ui32CurrentFLPages,
			psRPMContext->ui32TotalRPMPages,
			psRPMContext->ui32UnallocatedPages));

	return PVRSRV_OK;

	/* Error handling */
ErrorFreeListWriteEntries:
	/* TODO: unmap sparse block for RPM FPL */
ErrorFreeListBlock:
	/* TODO: unmap sparse block for RPM page table */
ErrorPageTableBlock:
	/* TODO: unmap sparse block for scene hierarchy */

ErrorSceneBlock:	
	OSLockRelease(psFreeList->psDevInfo->hLockRPMContext);
	OSLockRelease(psFreeList->psDevInfo->hLockRPMFreeList);
	OSFreeMem(psRPMDevMemDesc);

	PVR_ASSERT(eError != PVRSRV_OK);
	return eError;
}

static PVRSRV_ERROR RGXShrinkRPMFreeList(PDLLIST_NODE pListHeader,
										 RGX_RPM_FREELIST *psFreeList)
{
	DLLIST_NODE *psNode;
	RGX_RPM_DEVMEM_DESC	*psRPMDevMemNode;
	PVRSRV_ERROR eError = PVRSRV_OK;
	IMG_UINT32 ui32OldValue;

	/*
	 * Lock protects simultaneous manipulation of:
	 * - the memory block list
	 * - the freelist's ui32CurrentFLPages value
	 */
	PVR_ASSERT(pListHeader);
	PVR_ASSERT(psFreeList);
	PVR_ASSERT(psFreeList->psDevInfo);
	PVR_ASSERT(psFreeList->psDevInfo->hLockRPMFreeList);

	OSLockAcquire(psFreeList->psDevInfo->hLockRPMFreeList);

	/********************************************************************
	 * All scene memory blocks must be freed together as non-contiguous
	 * virtual mappings are not yet supported.
	 ********************************************************************/

	/* Get node from head of list and remove it */
	psNode = dllist_get_next_node(pListHeader);
	PVR_DPF((PVR_DBG_MESSAGE, "Found node %p", psNode));
	if (psNode)
	{
		dllist_remove_node(psNode);

		psRPMDevMemNode = IMG_CONTAINER_OF(psNode, RGX_RPM_DEVMEM_DESC, sMemoryDescBlock);
		PVR_ASSERT(psRPMDevMemNode);
		PVR_ASSERT(psRPMDevMemNode->psFreeList);
		PVR_ASSERT(psRPMDevMemNode->sSceneHierarchyNode.psPMR);

		/* remove scene hierarchy block */
		PVR_DPF((PVR_DBG_MESSAGE, "Removing scene hierarchy node"));
		eError = _RGXUnmapRPMPBBlock(&psRPMDevMemNode->sSceneHierarchyNode,
									 psRPMDevMemNode->psFreeList,
									 psFreeList->psParentCtx->sSceneMemoryBaseAddr);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR, "RGXShrinkRPMFreeList: Failed to unmap %d pages with mapping index %d (status %d)",
					psRPMDevMemNode->sSceneHierarchyNode.ui32NumPhysPages,
					psRPMDevMemNode->sSceneHierarchyNode.ui32StartOfMappingIndex,
					eError));
			goto UnMapError;
		}

		/* 
		 * If the grow size is sub OS page size then the page lists may not need updating 
		 */
		if (psRPMDevMemNode->sRPMPageListNode.eNodeType != NODE_EMPTY)
		{
			/* unmap the RPM page table backing pages */
			PVR_DPF((PVR_DBG_MESSAGE, "Removing RPM page list node"));
			PVR_ASSERT(psRPMDevMemNode->sRPMPageListNode.psPMR);
			eError = _RGXUnmapRPMPBBlock(&psRPMDevMemNode->sRPMPageListNode,
										 psRPMDevMemNode->psFreeList,
										 psFreeList->psParentCtx->sRPMPageTableBaseAddr);
			if (eError != PVRSRV_OK)
			{
				PVR_DPF((PVR_DBG_ERROR, "RGXShrinkRPMFreeList: Failed to unmap %d pages with mapping index %d (status %d)",
						psRPMDevMemNode->sRPMPageListNode.ui32NumPhysPages,
						psRPMDevMemNode->sRPMPageListNode.ui32StartOfMappingIndex,
						eError));
				goto UnMapError;
			}
		}

		if (psRPMDevMemNode->sRPMFreeListNode.eNodeType != NODE_EMPTY)
		{
			/* unmap the RPM free page list backing pages */
			PVR_DPF((PVR_DBG_MESSAGE, "Removing RPM free list node"));
			PVR_ASSERT(psRPMDevMemNode->sRPMFreeListNode.psPMR);
			eError = _RGXUnmapRPMPBBlock(&psRPMDevMemNode->sRPMFreeListNode,
										 psRPMDevMemNode->psFreeList,
										 psFreeList->sBaseDevVAddr);
			if (eError != PVRSRV_OK)
			{
				PVR_DPF((PVR_DBG_ERROR, "RGXShrinkRPMFreeList: Failed to unmap %d pages with mapping index %d (status %d)",
						psRPMDevMemNode->sRPMFreeListNode.ui32NumPhysPages,
						psRPMDevMemNode->sRPMFreeListNode.ui32StartOfMappingIndex,
						eError));
				goto UnMapError;
			}
		}

		/* update available RPM pages in freelist (NOTE: may be different from phys page count) */
		ui32OldValue = psFreeList->ui32CurrentFLPages;
		psFreeList->ui32CurrentFLPages -= psRPMDevMemNode->ui32NumPages;

		/* check underflow */
		PVR_ASSERT(ui32OldValue > psFreeList->ui32CurrentFLPages);

		PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p, ID %d]: shrink by %u pages (current pages %u/%u)",
								psFreeList,
								psFreeList->ui32FreelistID,
								psRPMDevMemNode->ui32NumPages,
								psFreeList->ui32CurrentFLPages,
								psFreeList->psParentCtx->ui32UnallocatedPages));

		OSFreeMem(psRPMDevMemNode);
	}
	else
	{
		PVR_DPF((PVR_DBG_WARNING,"Freelist [0x%p]: shrink denied. PB already at zero PB size (%u pages)",
								psFreeList,
								psFreeList->ui32CurrentFLPages));
		eError = PVRSRV_ERROR_PBSIZE_ALREADY_MIN;
	}

	OSLockRelease(psFreeList->psDevInfo->hLockRPMFreeList);
	return PVRSRV_OK;

UnMapError:
	OSFreeMem(psRPMDevMemNode);
	OSLockRelease(psFreeList->psDevInfo->hLockRPMFreeList);

	PVR_ASSERT(eError != PVRSRV_OK);
	return eError;
}


/*!
 *	_RGXCreateRPMSparsePMR
 * 
 * Creates a PMR container with no phys pages initially. Phys pages will be allocated
 * and mapped later when requested by client or by HW RPM Out of Memory event.
 * The PMR is created with zero phys backing pages.
 * The sparse PMR is associated to either the RPM context or to the RPM freelist(s):
 * 
 * RGX_SERVER_RPM_CONTEXT - Scene hierarchy, page table
 * RGX_RPM_FREELIST - free page list PMR
 * 
 * @param	eBlockType - whether block is for scene hierarchy pages or page
 * 				tables. This parameter is used to calculate size.
 * @param	ui32NumPages - total number of pages
 * @param	uiLog2DopplerPageSize - log2 Doppler/RPM page size
 * @param	ppsPMR - (Output) new PMR container.
 * 
 * See the documentation for more details.
 */
static
PVRSRV_ERROR _RGXCreateRPMSparsePMR(CONNECTION_DATA *psConnection,
									PVRSRV_DEVICE_NODE	 *psDeviceNode,
									RGX_DEVMEM_NODE_TYPE eBlockType,
									IMG_UINT32		ui32NumPages,
									IMG_UINT32		uiLog2DopplerPageSize,
									PMR				**ppsPMR)
{
	PVRSRV_ERROR		eError;
	IMG_DEVMEM_SIZE_T	uiMaxSize = 0;
	IMG_UINT32			ui32NumVirtPages = 0; /*!< number of virtual pages to cover virtual range */
	IMG_UINT32			ui32Log2OSPageSize = OSGetPageShift();
	IMG_UINT32			ui32ChunkSize = OSGetPageSize();
	PVRSRV_MEMALLOCFLAGS_T uiCustomFlags = 0;

	/* Work out the allocation logical size = virtual size */
	switch(eBlockType)
	{
		case NODE_EMPTY:
			PVR_ASSERT(IMG_FALSE);
			return PVRSRV_ERROR_INVALID_PARAMS;
		case NODE_SCENE_HIERARCHY:
			PDUMPCOMMENT("Allocate Scene Hierarchy PMR (Pages %08X)", ui32NumPages);
			uiMaxSize = (IMG_DEVMEM_SIZE_T)ui32NumPages * (1 << uiLog2DopplerPageSize);
			break;
		case NODE_RPM_PAGE_TABLE:
			PDUMPCOMMENT("Allocate RPM Page Table PMR (Page entries %08X)", ui32NumPages);
			uiMaxSize = (IMG_DEVMEM_SIZE_T)ui32NumPages * sizeof(RGX_RPM_DATA_RTU_PAGE_TABLE);
			break;
		case NODE_RPM_FREE_PAGE_LIST:
			/* 
			 * Each RPM free page list (FPL) supports the maximum range.
			 * In practise the maximum range is divided between allocations in each FPL
			 */
			PDUMPCOMMENT("Allocate RPM Free Page List PMR (Page entries %08X)", ui32NumPages);
			uiMaxSize = (IMG_DEVMEM_SIZE_T)ui32NumPages * sizeof(RGX_RPM_DATA_RTU_FREE_PAGE_LIST);
			uiCustomFlags |= PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE; /*(PVRSRV_MEMALLOCFLAG_CPU_READABLE | PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | PVRSRV_MEMALLOCFLAG_CPU_UNCACHED); */
			break;
		/* no default case because the build should error out if a case is unhandled */
	}

	uiMaxSize = (uiMaxSize + ui32ChunkSize - 1) & ~(ui32ChunkSize - 1);
	ui32NumVirtPages = uiMaxSize >> ui32Log2OSPageSize;

	eError = PhysmemNewRamBackedPMR(psConnection,
									psDeviceNode,
									uiMaxSize, /* the maximum size which should match num virtual pages * page size */
									ui32ChunkSize,
									0,
									ui32NumVirtPages,
									NULL,
									ui32Log2OSPageSize,
									(PVRSRV_MEMALLOCFLAG_GPU_READABLE | PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | PVRSRV_MEMALLOCFLAG_SPARSE_NO_DUMMY_BACKING | uiCustomFlags),
									strlen("RPM Buffer") + 1,
									"RPM Buffer",
									ppsPMR);
	if(eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,
				 "_RGXCreateRPMSparsePMR: Failed to allocate sparse PMR of size: 0x%016llX",
				 (IMG_UINT64)uiMaxSize));
	}
	
	return eError;
}

/*!
 *	_RGXMapRPMPBBlock
 * 
 * Maps in a block of phys pages for one of the following:
 * 
 * NODE_SCENE_HIERARCHY - scene hierarchy
 * NODE_RPM_PAGE_TABLE - RPM page table entries
 * NODE_RPM_FREE_PAGE_LIST - RPM free page list entries
 * 
 * @param	psDevMemNode - device mem block descriptor (allocated by caller)
 * @param	psFreeList - free list descriptor
 * @param	eBlockType - block type: scene memory, RPM page table or RPM page free list
 * @param	psDevmemHeap - heap for GPU virtual mapping
 * @param	ui32NumPages - number of pages for scene memory, OR
 * 							number of PT entries for RPM page table or page free list
 * @param	sDevVAddrBase - GPU virtual base address i.e. base address at start of sparse allocation
 * 
 * @return	PVRSRV_OK if no error occurred
 */
static
PVRSRV_ERROR _RGXMapRPMPBBlock(RGX_DEVMEM_NODE	*psDevMemNode,
					RGX_RPM_FREELIST *psFreeList,
					RGX_DEVMEM_NODE_TYPE eBlockType,
					DEVMEMINT_HEAP *psDevmemHeap,
					IMG_UINT32 ui32NumPages,
					IMG_DEV_VIRTADDR sDevVAddrBase)
{
	PVRSRV_ERROR	eError;
    IMG_UINT64 		sCpuVAddrNULL = 0; 			/* no CPU mapping needed */
	IMG_UINT32		*paui32AllocPageIndices;	/* table of virtual indices for sparse mapping */
	IMG_PUINT32 	pui32MappingIndex = NULL;	/* virtual index where next physical chunk is mapped */
	IMG_UINT32		i;
	size_t			uiSize = 0;
	IMG_UINT32		ui32Log2OSPageSize = OSGetPageShift();
	IMG_UINT32		ui32ChunkSize = OSGetPageSize();
	IMG_UINT32		ui32NumPhysPages = 0; /*!< number of physical pages for data pages or RPM PTs */
	PVRSRV_MEMALLOCFLAGS_T uiCustomFlags = 0;


	/* Allocate Memory Block for scene hierarchy */
	switch(eBlockType)
	{
		case NODE_EMPTY:
			PVR_ASSERT(IMG_FALSE);
			return PVRSRV_ERROR_INVALID_PARAMS;
		case NODE_SCENE_HIERARCHY:
			PDUMPCOMMENT("Allocate Scene Hierarchy Block (Pages %08X)", ui32NumPages);
			uiSize = (size_t)ui32NumPages * (1 << psFreeList->psParentCtx->uiLog2DopplerPageSize);
			pui32MappingIndex = &psFreeList->psParentCtx->ui32SceneMemorySparseMappingIndex;
			break;
		case NODE_RPM_PAGE_TABLE:
			PDUMPCOMMENT("Allocate RPM Page Table Block (Page entries %08X)", ui32NumPages);
			uiSize = (size_t)ui32NumPages * sizeof(RGX_RPM_DATA_RTU_PAGE_TABLE);
			pui32MappingIndex = &psFreeList->psParentCtx->ui32RPMPageTableSparseMappingIndex;
			break;
		case NODE_RPM_FREE_PAGE_LIST:
			PDUMPCOMMENT("Allocate RPM Free Page List Block (Page entries %08X)", ui32NumPages);
			uiSize = (size_t)ui32NumPages * sizeof(RGX_RPM_DATA_RTU_FREE_PAGE_LIST);
			pui32MappingIndex = &psFreeList->ui32RPMFreeListSparseMappingIndex;
			uiCustomFlags |= PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE; /*(PVRSRV_MEMALLOCFLAG_CPU_READABLE | PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE);*/
			break;
		/* no default case because the build should error out if a case is unhandled */
	}

	/* 
	 * Round size up to multiple of the sparse chunk size = OS page size.
	 */
	uiSize = (uiSize + ui32ChunkSize - 1) & ~(ui32ChunkSize - 1);
	ui32NumPhysPages = uiSize >> ui32Log2OSPageSize;

	paui32AllocPageIndices = OSAllocMem(ui32NumPhysPages * sizeof(IMG_UINT32));
    if (paui32AllocPageIndices == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR, "_RGXCreateRPMPBBlockSparse: failed to allocate sparse mapping index list"));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto ErrorAllocHost;
	}
	for(i=0; i<ui32NumPhysPages; i++)
	{
		paui32AllocPageIndices[i] = *pui32MappingIndex + i;
	}

	/* Set up some state */
	psDevMemNode->eNodeType = eBlockType;
	psDevMemNode->psDevMemHeap = psDevmemHeap;
	if (eBlockType == NODE_SCENE_HIERARCHY)
	{
		/* the mapped-in scene hierarchy device address will be used to set up the FPL entries */
		psDevMemNode->sAddr.uiAddr = sDevVAddrBase.uiAddr + (*pui32MappingIndex * ui32ChunkSize);
	}
	psDevMemNode->ui32NumPhysPages = ui32NumPhysPages;
	psDevMemNode->ui32StartOfMappingIndex = *pui32MappingIndex;

	{
		if ((eBlockType == NODE_SCENE_HIERARCHY) &&
			(ui32NumPhysPages > psFreeList->psParentCtx->ui32UnallocatedPages))
		{
			PVR_DPF((PVR_DBG_ERROR, "_RGXCreateRPMPBBlockSparse: virtual address space exceeded (0x%x pages required, 0x%x pages available).",
					ui32NumPhysPages, psFreeList->psParentCtx->ui32UnallocatedPages));
			OSFreeMem(paui32AllocPageIndices);
			return PVRSRV_ERROR_INVALID_PARAMS;
		}

		eError = PMRLockSysPhysAddresses(psDevMemNode->psPMR);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR, "_RGXCreateRPMPBBlockSparse: unable to lock PMR physical pages (status %d)", eError));
			goto ErrorLockPhys;
		}

		eError = DevmemIntChangeSparse(psDevmemHeap,
						psDevMemNode->psPMR,
						ui32NumPhysPages,
						paui32AllocPageIndices,
						0,
						NULL,
						SPARSE_RESIZE_ALLOC,
						(PVRSRV_MEMALLOCFLAG_GPU_READABLE | PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | uiCustomFlags),
						sDevVAddrBase,
						sCpuVAddrNULL);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR, "_RGXCreateRPMPBBlockSparse: change sparse mapping failed with %d pages starting at %d (status %d)",
					ui32NumPhysPages, *pui32MappingIndex, eError));
			goto ErrorSparseMapping;
		}

		/* FIXME: leave locked until destroy */
		PMRUnlockSysPhysAddresses(psDevMemNode->psPMR);
	}

	/* 
	 * Update the mapping index for the next allocation.
	 * The virtual pages should be contiguous.
	 */
	*pui32MappingIndex += ui32NumPhysPages;

	OSFreeMem(paui32AllocPageIndices);

	return PVRSRV_OK;

ErrorSparseMapping:
	PMRUnlockSysPhysAddresses(psDevMemNode->psPMR);

ErrorLockPhys:
	OSFreeMem(paui32AllocPageIndices);

ErrorAllocHost:
	PVR_ASSERT(eError != PVRSRV_OK);
	return eError;
}

/*!
 * _RGXUnmapRPMPBBlock
 * 
 * NOTE: because the SHF and SHG requests for memory are interleaved, the
 * page mapping offset cannot be updated (non-contiguous virtual mapping
 * is not supported).
 * 
 * So either
 *  (i) the allocated virtual address range is unusable after unmap
 * (ii) all of the scene memory must be freed
 * 
 * @param	psDevMemNode - block to free
 * @param	psFreeList - RPM free list
 * @param	sDevVAddrBase - the virtual base address (i.e. where page 1 of the PMR is mapped)
 */
static
PVRSRV_ERROR _RGXUnmapRPMPBBlock(RGX_DEVMEM_NODE	*psDevMemNode,
					RGX_RPM_FREELIST *psFreeList,
					IMG_DEV_VIRTADDR sDevVAddrBase)
{
	PVRSRV_ERROR	eError;
	IMG_UINT64 		sCpuVAddrNULL = 0; 			/* no CPU mapping needed */
	IMG_UINT32		*paui32FreePageIndices;		/* table of virtual indices for sparse unmapping */
	IMG_UINT32		i;
	IMG_UINT32		ui32NumPhysPages = psDevMemNode->ui32NumPhysPages; /*!< number of physical pages for data pages or RPM PTs */

#if defined(PDUMP)
	/* Free Memory Block for scene hierarchy */
	switch(psDevMemNode->eNodeType)
	{
		case NODE_EMPTY:
			PVR_ASSERT(IMG_FALSE);
			return PVRSRV_ERROR_INVALID_PARAMS;
		case NODE_SCENE_HIERARCHY:
			PDUMPCOMMENT("Free Scene Hierarchy Block (Pages %08X)", ui32NumPhysPages);
			break;
		case NODE_RPM_PAGE_TABLE:
			PDUMPCOMMENT("Free RPM Page Table Block (Page entries %08X)", ui32NumPhysPages);
			break;
		case NODE_RPM_FREE_PAGE_LIST:
			PDUMPCOMMENT("Free RPM Free Page List Block (Page entries %08X)", ui32NumPhysPages);
			break;
		/* no default case because the build should error out if a case is unhandled */
	}
#endif

	paui32FreePageIndices = OSAllocMem(ui32NumPhysPages * sizeof(IMG_UINT32));
    if (paui32FreePageIndices == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR, "_RGXUnmapRPMPBBlock: failed to allocate sparse mapping index list"));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto ErrorAllocHost;
	}
	for(i=0; i<ui32NumPhysPages; i++)
	{
		paui32FreePageIndices[i] = psDevMemNode->ui32StartOfMappingIndex + i;
	}

	{
		eError = PMRLockSysPhysAddresses(psDevMemNode->psPMR);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR, "_RGXUnmapRPMPBBlock: unable to lock PMR physical pages (status %d)", eError));
			goto ErrorLockPhys;
		}

		eError = DevmemIntChangeSparse(psDevMemNode->psDevMemHeap,
						psDevMemNode->psPMR,
						0, /* no pages are mapped here */
						NULL,
						ui32NumPhysPages,
						paui32FreePageIndices,
						SPARSE_RESIZE_FREE,
						(PVRSRV_MEMALLOCFLAG_GPU_READABLE | PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE),
						sDevVAddrBase,
						sCpuVAddrNULL);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR, "_RGXUnmapRPMPBBlock: free sparse mapping failed with %d pages starting at %d (status %d)",
					ui32NumPhysPages, psDevMemNode->ui32StartOfMappingIndex, eError));
			goto ErrorSparseMapping;
		}

		PMRUnlockSysPhysAddresses(psDevMemNode->psPMR);
	}

	OSFreeMem(paui32FreePageIndices);

	return PVRSRV_OK;

ErrorSparseMapping:
	PMRUnlockSysPhysAddresses(psDevMemNode->psPMR);

ErrorLockPhys:
	OSFreeMem(paui32FreePageIndices);

ErrorAllocHost:
	PVR_ASSERT(eError != PVRSRV_OK);
	return eError;
}


/*!
 *	RGXCreateRPMFreeList
 * 
 * @param	ui32InitFLPages - initial allocation of mapped-in physical pages
 * @param	ui32GrowFLPages - physical pages to add to scene hierarchy if RPM OOM occurs
 * @param	sFreeListDevVAddr - virtual base address of free list
 * @param	sRPMPageListDevVAddr (DEPRECATED -- cached in RPM Context)
 * @param	ui32FLSyncAddr (DEPRECATED)
 * @param	ppsFreeList - returns a RPM freelist handle to client
 * @param	puiHWFreeList - 'handle' to FW freelist, passed in VRDM kick (FIXME)
 * @param	bIsExternal - flag which marks if the freelist is an external one
 */
IMG_EXPORT
PVRSRV_ERROR RGXCreateRPMFreeList(CONNECTION_DATA *psConnection,
							   PVRSRV_DEVICE_NODE	 *psDeviceNode, 
							   RGX_SERVER_RPM_CONTEXT	*psRPMContext,
							   IMG_UINT32			ui32InitFLPages,
							   IMG_UINT32			ui32GrowFLPages,
							   IMG_DEV_VIRTADDR		sFreeListDevVAddr,
							   RGX_RPM_FREELIST	  **ppsFreeList,
							   IMG_UINT32		   *puiHWFreeList,
							   IMG_BOOL				bIsExternal)
{
	PVRSRV_ERROR				eError;
	RGXFWIF_RPM_FREELIST		*psFWRPMFreeList;
	DEVMEM_MEMDESC				*psFWRPMFreelistMemDesc;
	RGX_RPM_FREELIST			*psFreeList;
	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;

	/* Allocate kernel freelist struct */
	psFreeList = OSAllocZMem(sizeof(*psFreeList));
    if (psFreeList == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXCreateRPMFreeList: failed to allocate host data structure"));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto ErrorAllocHost;
	}

	/* Allocate cleanup sync */
	eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
						   &psFreeList->psCleanupSync,
						   "RPM free list cleanup");
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"RGXCreateRPMFreeList: Failed to allocate cleanup sync (0x%x)",
				eError));
		goto ErrorSyncAlloc;
	}

	/*
	 * This FW FreeList context is only mapped into kernel for initialisation.
	 * Otherwise this allocation is only used by the FW.
	 * Therefore the GPU cache doesn't need coherency,
	 * and write-combine is suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
	 * 
	 * TODO - RPM freelist will be modified after creation, but only from host-side.
	 */
	eError = DevmemFwAllocate(psDevInfo,
							sizeof(*psFWRPMFreeList),
							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
							PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
							PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
							PVRSRV_MEMALLOCFLAG_GPU_READABLE |
							PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
							PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
							PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
							PVRSRV_MEMALLOCFLAG_CPU_READABLE |
							PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
							"FwRPMFreeList",
							&psFWRPMFreelistMemDesc);
	if (eError != PVRSRV_OK) 
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXCreateRPMFreeList: DevmemAllocate for RGXFWIF_FREELIST failed"));
		goto ErrorFWFreeListAlloc;
	}

	/* Initialise host data structures */
	psFreeList->psConnection = psConnection;
	psFreeList->psDevInfo = psDevInfo;
	psFreeList->psParentCtx = psRPMContext;
	psFreeList->psFWFreelistMemDesc = psFWRPMFreelistMemDesc;
	psFreeList->sBaseDevVAddr = sFreeListDevVAddr;
	RGXSetFirmwareAddress(&psFreeList->sFreeListFWDevVAddr, psFWRPMFreelistMemDesc, 0, RFW_FWADDR_FLAG_NONE);
	psFreeList->ui32FreelistID = psDevInfo->ui32RPMFreelistCurrID++;
	//psFreeList->ui32MaxFLPages = ui32MaxFLPages;
	/* TODO: is it really needed? */
	if(bIsExternal == IMG_FALSE)
	{
		psFreeList->ui32InitFLPages = ui32InitFLPages;
		psFreeList->ui32GrowFLPages = ui32GrowFLPages;
	}
	//psFreeList->ui32CurrentFLPages = ui32InitFLPages;
	psFreeList->ui32RefCount = 0;
	dllist_init(&psFreeList->sMemoryBlockHead);

	/* Wizard2 -- support per-freelist Doppler virtual page size */
	psFreeList->uiLog2DopplerPageSize = psRPMContext->uiLog2DopplerPageSize;

	/* Initialise FW data structure */
	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWRPMFreeList);
	PVR_LOGG_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", ErrorFWFreeListCpuMap);

	/*
	 * FIXME - the max pages are shared with the other freelists so this
	 * over-estimates the number of free pages. The full check is
	 * implemented in RGXGrowRPMFreeList.
	 */
	if(bIsExternal == IMG_TRUE)
	{
		/* An external RPM FreeList will never grow */
		psFWRPMFreeList->ui32MaxPages = ui32InitFLPages;
	}
	else
	{
		psFWRPMFreeList->ui32MaxPages = psFreeList->psParentCtx->ui32TotalRPMPages;
	}
	psFWRPMFreeList->ui32CurrentPages = ui32InitFLPages;
	psFWRPMFreeList->ui32GrowPages = ui32GrowFLPages;
	psFWRPMFreeList->ui32ReadOffset = 0;
	psFWRPMFreeList->ui32WriteOffset = RGX_CR_RPM_SHG_FPL_WRITE_TOGGLE_EN; /* FL is full */
	psFWRPMFreeList->bReadToggle = IMG_FALSE;
	psFWRPMFreeList->bWriteToggle = IMG_TRUE;
	psFWRPMFreeList->sFreeListDevVAddr.uiAddr = sFreeListDevVAddr.uiAddr;
	psFWRPMFreeList->ui32FreeListID = psFreeList->ui32FreelistID;
	psFWRPMFreeList->bGrowPending = IMG_FALSE;

	PVR_DPF((PVR_DBG_MESSAGE, "RPM Freelist %p created: FW freelist: %p, Init pages 0x%08x, Max FL base address " IMG_DEVMEM_SIZE_FMTSPEC ", Init FL base address " IMG_DEVMEM_SIZE_FMTSPEC,
			psFreeList,
			psFWRPMFreeList,
			ui32InitFLPages,
			sFreeListDevVAddr.uiAddr,
			psFWRPMFreeList->sFreeListDevVAddr.uiAddr));

	PVR_DPF((PVR_DBG_MESSAGE,"RPM FW Freelist %p created: sync FW addr 0x%08x", psFWRPMFreeList, psFWRPMFreeList->sSyncAddr));

	PDUMPCOMMENT("Dump FW RPM FreeList");
	DevmemPDumpLoadMem(psFreeList->psFWFreelistMemDesc, 0, sizeof(*psFWRPMFreeList), PDUMP_FLAGS_CONTINUOUS);

	/*
	 * Separate dump of the Freelist's number of Pages and stack pointer.
	 * This allows to easily modify the PB size in the out2.txt files.
	 */
	PDUMPCOMMENT("RPM FreeList TotalPages");
	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
							offsetof(RGXFWIF_RPM_FREELIST, ui32CurrentPages),
							psFWRPMFreeList->ui32CurrentPages,
							PDUMP_FLAGS_CONTINUOUS);

	PDUMPCOMMENT("RPM FreeList device virtual base address");
	DevmemPDumpLoadMemValue64(psFreeList->psFWFreelistMemDesc,
							offsetof(RGXFWIF_RPM_FREELIST, sFreeListDevVAddr),
							psFWRPMFreeList->sFreeListDevVAddr.uiAddr,
							PDUMP_FLAGS_CONTINUOUS);

	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);

	if (bIsExternal == IMG_TRUE)
	{
		/* Mark the freelist as an external */
		psFreeList->bIsExternal = IMG_TRUE;

		/* In case of an external RPM FreeList it is not needed to:
		 * 		- create sparse PMR
		 * 		- allocate physical memory for the freelist
		 * 		- add it to the list of freelist
		 */

		/* return values */
		*puiHWFreeList = psFreeList->sFreeListFWDevVAddr.ui32Addr;
		*ppsFreeList = psFreeList;

		return PVRSRV_OK;
	}

	psFreeList->bIsExternal = IMG_FALSE;

	/*
	 * Create the sparse PMR for the RPM free page list
	 */
	eError = _RGXCreateRPMSparsePMR(psConnection, psDeviceNode,
									NODE_RPM_FREE_PAGE_LIST,
									psRPMContext->ui32TotalRPMPages,
									psRPMContext->uiLog2DopplerPageSize,
									&psFreeList->psFreeListPMR);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXCreateRPMContext: failed to allocate PMR for RPM Free page list (%d)", eError));
		goto ErrorSparsePMR;
	}

	/*
	 * Lock protects simultaneous manipulation of:
	 * - the memory block list
	 * - the freelist's ui32CurrentFLPages
	 */
	/* Add to list of freelists */
	OSLockAcquire(psDevInfo->hLockRPMFreeList);
	psFreeList->psParentCtx->uiFLRefCount++;
	dllist_add_to_tail(&psDevInfo->sRPMFreeListHead, &psFreeList->sNode);
	OSLockRelease(psDevInfo->hLockRPMFreeList);

	/*
	 * Add initial scene hierarchy block
	 * Allocate phys memory for scene hierarchy, free page list and RPM page-in-use list
	 */
	eError = RGXGrowRPMFreeList(psFreeList, ui32InitFLPages, &psFreeList->sMemoryBlockHead);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXCreateRPMFreeList: error during phys memory allocation and mapping (%d)", eError));
		goto ErrorGrowFreeList;
	}

	/* return values */
	*puiHWFreeList = psFreeList->sFreeListFWDevVAddr.ui32Addr;
	*ppsFreeList = psFreeList;

	return PVRSRV_OK;

	/* Error handling */
ErrorGrowFreeList:
	/* Remove freelists from list  */
	OSLockAcquire(psDevInfo->hLockRPMFreeList);
	dllist_remove_node(&psFreeList->sNode);
	psFreeList->psParentCtx->uiFLRefCount--;
	OSLockRelease(psDevInfo->hLockRPMFreeList);

ErrorSparsePMR:
	SyncPrimFree(psFreeList->psCleanupSync);

ErrorFWFreeListCpuMap:
	RGXUnsetFirmwareAddress(psFWRPMFreelistMemDesc);
	DevmemFwFree(psDevInfo, psFWRPMFreelistMemDesc);

ErrorFWFreeListAlloc:
	PMRUnrefPMR(psFreeList->psFreeListPMR);

ErrorSyncAlloc:
	OSFreeMem(psFreeList);

ErrorAllocHost:
	PVR_ASSERT(eError != PVRSRV_OK);
	return eError;
}

/*
 *	RGXDestroyRPMFreeList
 */
IMG_EXPORT
PVRSRV_ERROR RGXDestroyRPMFreeList(RGX_RPM_FREELIST *psFreeList)
{
	PVRSRV_ERROR eError;
	//IMG_UINT64 ui64CheckSum;

	PVR_ASSERT(psFreeList);

	if(psFreeList->ui32RefCount != 0 && psFreeList->bIsExternal == IMG_FALSE)
	{
		/* Freelist still busy */
		PVR_DPF((PVR_DBG_WARNING, "Freelist %p is busy", psFreeList));
		return PVRSRV_ERROR_RETRY;
	}

	/* Freelist is not in use => start firmware cleanup */
	eError = RGXFWRequestRPMFreeListCleanUp(psFreeList->psDevInfo,
											psFreeList->sFreeListFWDevVAddr,
											psFreeList->psCleanupSync);
	if(eError != PVRSRV_OK)
	{
		/* Can happen if the firmware took too long to handle the cleanup request,
		 * or if SLC-flushes didn't went through (due to some GPU lockup) */
		return eError;
	}

	/* update the statistics */
#if defined(PVRSRV_ENABLE_PROCESS_STATS)
	PVRSRVStatsUpdateFreelistStats(psFreeList->ui32NumGrowReqByApp,
	                               psFreeList->ui32NumGrowReqByFW,
	                               psFreeList->ui32InitFLPages,
	                               psFreeList->ui32NumHighPages,
	                               0); /* FIXME - owner PID */
#endif

	/* Destroy FW structures */
	RGXUnsetFirmwareAddress(psFreeList->psFWFreelistMemDesc);
	DevmemFwFree(psFreeList->psDevInfo, psFreeList->psFWFreelistMemDesc);

	if(psFreeList->bIsExternal == IMG_FALSE)
	{
		/* Free the phys mem block descriptors. */
		PVR_DPF((PVR_DBG_WARNING, "Cleaning RPM freelist index %d", psFreeList->ui32FreelistID));
		while (!dllist_is_empty(&psFreeList->sMemoryBlockHead))
		{
			eError = RGXShrinkRPMFreeList(&psFreeList->sMemoryBlockHead, psFreeList);
			PVR_ASSERT(eError == PVRSRV_OK);
		}
		psFreeList->psParentCtx->uiFLRefCount--;

		/* consistency checks */
		PVR_ASSERT(dllist_is_empty(&psFreeList->sMemoryBlockHead));
		PVR_ASSERT(psFreeList->ui32CurrentFLPages == 0);

		/* Free RPM Free page list PMR */
		eError = PMRUnrefPMR(psFreeList->psFreeListPMR);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR,
					 "RGXDestroyRPMFreeList: Failed to free RPM free page list PMR %p (error %u)",
					 psFreeList->psFreeListPMR,
					 eError));
			PVR_ASSERT(IMG_FALSE);
		}

		/* Remove RPM FreeList from list */
		OSLockAcquire(psFreeList->psDevInfo->hLockRPMFreeList);
		dllist_remove_node(&psFreeList->sNode);
		OSLockRelease(psFreeList->psDevInfo->hLockRPMFreeList);
	}

	SyncPrimFree(psFreeList->psCleanupSync);

	/* free Freelist */
	OSFreeMem(psFreeList);

	return eError;
}


/*!
 *	RGXAddBlockToRPMFreeListKM
 * 
 * NOTE: This API isn't used but it's provided for symmetry with the parameter
 * management API.
*/
IMG_EXPORT
PVRSRV_ERROR RGXAddBlockToRPMFreeListKM(RGX_RPM_FREELIST *psFreeList,
										IMG_UINT32 ui32NumPages)
{
	PVRSRV_ERROR eError;

	/* Check if we have reference to freelist's PMR */
	if (psFreeList->psFreeListPMR == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR,	"RPM Freelist is not configured for grow"));
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	/* grow freelist */
	eError = RGXGrowRPMFreeList(psFreeList,
								ui32NumPages,
								&psFreeList->sMemoryBlockHead);
	if(eError == PVRSRV_OK)
	{
		/* update freelist data in firmware */
		_UpdateFwRPMFreelistSize(psFreeList, IMG_TRUE, IMG_TRUE, ui32NumPages);

		psFreeList->ui32NumGrowReqByApp++;

#if defined(PVRSRV_ENABLE_PROCESS_STATS)
			/* Update Stats */
			PVRSRVStatsUpdateFreelistStats(1, /* Add 1 to the appropriate counter (Requests by App)*/
	                               0,
	                               psFreeList->ui32InitFLPages,
	                               psFreeList->ui32NumHighPages,
	                               psFreeList->ownerPid);

#endif
	}

	return eError;
}


/*
 * RGXCreateRPMContext
 */
IMG_EXPORT
PVRSRV_ERROR RGXCreateRPMContext(CONNECTION_DATA * psConnection,
								 PVRSRV_DEVICE_NODE	 *psDeviceNode, 
								 RGX_SERVER_RPM_CONTEXT	**ppsRPMContext,
								 IMG_UINT32			ui32TotalRPMPages,
								 IMG_UINT32			uiLog2DopplerPageSize,
								 IMG_DEV_VIRTADDR	sSceneMemoryBaseAddr,
								 IMG_DEV_VIRTADDR	sDopplerHeapBaseAddr,
								 DEVMEMINT_HEAP		*psSceneHeap,
								 IMG_DEV_VIRTADDR	sRPMPageTableBaseAddr,
								 DEVMEMINT_HEAP		*psRPMPageTableHeap,
								 DEVMEM_MEMDESC		**ppsMemDesc,
							     IMG_UINT32		     *puiHWFrameData)
{
	PVRSRV_ERROR					eError;
	PVRSRV_RGXDEV_INFO 				*psDevInfo = psDeviceNode->pvDevice;
	//DEVMEM_MEMDESC				*psFWRPMContextMemDesc;
	RGX_SERVER_RPM_CONTEXT			*psRPMContext;
	RGXFWIF_RAY_FRAME_DATA			*psFrameData;
	RGXFWIF_DEV_VIRTADDR 			 sFirmwareAddr;

	/* Allocate kernel RPM context */
	psRPMContext = OSAllocZMem(sizeof(*psRPMContext));
    if (psRPMContext == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXCreateRPMContext: failed to allocate host data structure"));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto ErrorAllocHost;
	}

	*ppsRPMContext = psRPMContext;

	/* Allocate cleanup sync */
	eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
						   &psRPMContext->psCleanupSync,
						   "RPM context cleanup");
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"RGXCreateRPMContext: Failed to allocate cleanup sync (0x%x)",
				eError));
		goto ErrorSyncAlloc;
	}

	/*
	 * 1. Create the sparse PMR for scene hierarchy
	 */
	eError = _RGXCreateRPMSparsePMR(psConnection, psDeviceNode,
									NODE_SCENE_HIERARCHY,
									ui32TotalRPMPages,
									uiLog2DopplerPageSize,
									&psRPMContext->psSceneHierarchyPMR);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXCreateRPMContext: failed to allocate PMR for Scene hierarchy (%d)", eError));
		goto ErrorSparsePMR1;
	}

	/*
	 * 2. Create the sparse PMR for the RPM page list
	 */
	eError = _RGXCreateRPMSparsePMR(psConnection, psDeviceNode,
									NODE_RPM_PAGE_TABLE,
									ui32TotalRPMPages,
									uiLog2DopplerPageSize,
									&psRPMContext->psRPMPageTablePMR);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXCreateRPMContext: failed to allocate PMR for RPM Page list (%d)", eError));
		goto ErrorSparsePMR2;
	}

	/* Allocate FW structure and return FW address to client */
	eError = DevmemFwAllocate(psDevInfo,
							sizeof(*psFrameData),
							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
							PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
							PVRSRV_MEMALLOCFLAG_GPU_READABLE |
							PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
							PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
							PVRSRV_MEMALLOCFLAG_CPU_READABLE |
							PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
							PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
							PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE,
							"FwRPMContext",
							ppsMemDesc);
	if (eError != PVRSRV_OK) 
	{
		PVR_DPF((PVR_DBG_ERROR, "RGXCreateRPMContext: DevmemAllocate for RGXFWIF_FREELIST failed"));
		goto ErrorFWRPMContextAlloc;
	}

	/* Update the unallocated pages, which are shared between the RPM freelists */
	psRPMContext->ui32UnallocatedPages = psRPMContext->ui32TotalRPMPages = ui32TotalRPMPages;
	psRPMContext->psDeviceNode = psDeviceNode;
	psRPMContext->psFWRPMContextMemDesc = *ppsMemDesc;
	psRPMContext->uiLog2DopplerPageSize = uiLog2DopplerPageSize;

	/* Cache the virtual alloc state for future phys page mapping */
	psRPMContext->sDopplerHeapBaseAddr = sDopplerHeapBaseAddr;
	psRPMContext->sSceneMemoryBaseAddr = sSceneMemoryBaseAddr;
	psRPMContext->psSceneHeap = psSceneHeap;
	psRPMContext->sRPMPageTableBaseAddr = sRPMPageTableBaseAddr;
	psRPMContext->psRPMPageTableHeap = psRPMPageTableHeap;

	/*
	 * TODO - implement RPM abort control using HW frame data to track
	 * abort status in RTU.
	 */
	RGXSetFirmwareAddress(&sFirmwareAddr, *ppsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
	*puiHWFrameData = sFirmwareAddr.ui32Addr;

	//eError = DevmemAcquireCpuVirtAddr(*ppsMemDesc, (void **)&psFrameData);
	//PVR_LOGG_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", ErrorFrameDataCpuMap);

	/*
	 * TODO: pdumping
	 */


	return PVRSRV_OK;

	/* Error handling */

	DevmemReleaseCpuVirtAddr(*ppsMemDesc);

ErrorFWRPMContextAlloc:
	PMRUnrefPMR(psRPMContext->psRPMPageTablePMR);

ErrorSparsePMR2:
	PMRUnrefPMR(psRPMContext->psSceneHierarchyPMR);

ErrorSparsePMR1:
	SyncPrimFree(psRPMContext->psCleanupSync);

ErrorSyncAlloc:
	OSFreeMem(psRPMContext);

ErrorAllocHost:
	PVR_ASSERT(eError != PVRSRV_OK);
	return eError;
}	


/*
 * RGXDestroyRPMContext
 */
IMG_EXPORT
PVRSRV_ERROR RGXDestroyRPMContext(RGX_SERVER_RPM_CONTEXT *psCleanupData)
{
	PVRSRV_ERROR				 eError;
	PVRSRV_RGXDEV_INFO 			*psDevInfo;
	PRGXFWIF_RAY_FRAME_DATA		 psFrameData;

	/* Wait for FW to process all commands */

	PVR_ASSERT(psCleanupData);

	RGXSetFirmwareAddress(&psFrameData, psCleanupData->psFWRPMContextMemDesc, 0, RFW_FWADDR_NOREF_FLAG);

	/* Cleanup frame data in SHG */
	eError = RGXFWRequestRayFrameDataCleanUp(psCleanupData->psDeviceNode,
										  psFrameData,
										  psCleanupData->psCleanupSync,
										  RGXFWIF_DM_SHG);
	if (eError == PVRSRV_ERROR_RETRY)
	{
		PVR_DPF((PVR_DBG_WARNING, "FrameData busy in SHG"));
		return eError;
	}

	psDevInfo = psCleanupData->psDeviceNode->pvDevice;

	/* Cleanup frame data in RTU */
	eError = RGXFWRequestRayFrameDataCleanUp(psCleanupData->psDeviceNode,
										  psFrameData,
										  psCleanupData->psCleanupSync,
										  RGXFWIF_DM_RTU);
	if (eError == PVRSRV_ERROR_RETRY)
	{
		PVR_DPF((PVR_DBG_WARNING, "FrameData busy in RTU"));
		return eError;
	}

	/* Free Scene hierarchy PMR (We should be the only one that holds a ref on the PMR) */
	eError = PMRUnrefPMR(psCleanupData->psSceneHierarchyPMR);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,
				 "RGXDestroyRPMContext: Failed to free scene hierarchy PMR %p (error %u)",
				 psCleanupData->psSceneHierarchyPMR,
				 eError));
		PVR_ASSERT(IMG_FALSE);
	}

	/* Free RPM Page list PMR */
	eError = PMRUnrefPMR(psCleanupData->psRPMPageTablePMR);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,
				 "RGXDestroyRPMContext: Failed to free RPM page list PMR %p (error %u)",
				 psCleanupData->psRPMPageTablePMR,
				 eError));
		PVR_ASSERT(IMG_FALSE);
	}

	if (psCleanupData->uiFLRefCount > 0)
	{
		/* Kernel RPM freelists hold reference to RPM context */
		PVR_DPF((PVR_DBG_WARNING, "RGXDestroyRPMContext: Free list ref count non-zero."));
		return PVRSRV_ERROR_NONZERO_REFCOUNT;
	}

	/* If we got here then SHG and RTU operations on this FrameData have finished */
	SyncPrimFree(psCleanupData->psCleanupSync);

	/* Free the FW RPM descriptor */
	RGXUnsetFirmwareAddress(psCleanupData->psFWRPMContextMemDesc);
	DevmemFwFree(psDevInfo, psCleanupData->psFWRPMContextMemDesc);

	OSFreeMem(psCleanupData);

	return PVRSRV_OK;
}


/*
 * PVRSRVRGXCreateRayContextKM
 */
IMG_EXPORT
PVRSRV_ERROR PVRSRVRGXCreateRayContextKM(CONNECTION_DATA				*psConnection,
											PVRSRV_DEVICE_NODE			*psDeviceNode,
											IMG_UINT32					ui32Priority,
											IMG_DEV_VIRTADDR			sMCUFenceAddr,
											IMG_DEV_VIRTADDR			sVRMCallStackAddr,
											IMG_UINT32					ui32FrameworkRegisterSize,
											IMG_PBYTE					pabyFrameworkRegisters,
											IMG_HANDLE					hMemCtxPrivData,
											RGX_SERVER_RAY_CONTEXT	**ppsRayContext)
{
	PVRSRV_ERROR				eError;
	PVRSRV_RGXDEV_INFO 			*psDevInfo = psDeviceNode->pvDevice;
	RGX_SERVER_RAY_CONTEXT		*psRayContext;
	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
	RGX_COMMON_CONTEXT_INFO		sInfo;
	RGXFWIF_FWRAYCONTEXT		*pFWRayContext;
	IMG_UINT32 i;

	/* Prepare cleanup structure */
    *ppsRayContext= NULL;
	psRayContext = OSAllocZMem(sizeof(*psRayContext));
    if (psRayContext == NULL)
	{
		return PVRSRV_ERROR_OUT_OF_MEMORY;
	}

	psRayContext->psDeviceNode = psDeviceNode;

	/*
		Allocate device memory for the firmware ray context.
	*/
	PDUMPCOMMENT("Allocate RGX firmware ray context");

	eError = DevmemFwAllocate(psDevInfo,
							sizeof(RGXFWIF_FWRAYCONTEXT),
							RGX_FWCOMCTX_ALLOCFLAGS,
							"FwRayContext",
							&psRayContext->psFWRayContextMemDesc);

	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRayContextKM: Failed to allocate firmware ray context (%u)",
				eError));
		goto fail_fwraycontext;
	}
					   
	/* Allocate cleanup sync */
	eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
						   &psRayContext->psCleanupSync,
						   "Ray context cleanup");
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRayContextKM: Failed to allocate cleanup sync (0x%x)",
				eError));
		goto fail_syncalloc;
	}
	
	/* 
	 * Create the FW framework buffer
	 */
	eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode, &psRayContext->psFWFrameworkMemDesc, ui32FrameworkRegisterSize);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRayContextKM: Failed to allocate firmware GPU framework state (%u)",
				eError));
		goto fail_frameworkcreate;
	}
	
	/* Copy the Framework client data into the framework buffer */
	eError = PVRSRVRGXFrameworkCopyCommand(psRayContext->psFWFrameworkMemDesc, pabyFrameworkRegisters, ui32FrameworkRegisterSize);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRayContextKM: Failed to populate the framework buffer (%u)",
				eError));
		goto fail_frameworkcopy;
	}

	sInfo.psFWFrameworkMemDesc = psRayContext->psFWFrameworkMemDesc;
	sInfo.psMCUFenceAddr = &sMCUFenceAddr;
	
	eError = _CreateSHContext(psConnection,
							  psDeviceNode,
							  psRayContext->psFWRayContextMemDesc,
							  offsetof(RGXFWIF_FWRAYCONTEXT, sSHGContext),
							  psFWMemContextMemDesc,
							  sVRMCallStackAddr,
							  ui32Priority,
							  &sInfo,
							  &psRayContext->sSHData);
	if (eError != PVRSRV_OK)
	{
		goto fail_shcontext;
	}

	eError = _CreateRSContext(psConnection,
							  psDeviceNode,
							  psRayContext->psFWRayContextMemDesc,
							  offsetof(RGXFWIF_FWRAYCONTEXT, sRTUContext),
							  psFWMemContextMemDesc,
							  ui32Priority,
							  &sInfo,
							  &psRayContext->sRSData);
	if (eError != PVRSRV_OK)
	{
		goto fail_rscontext;
	}
		
	/*
		Temporarily map the firmware context to the kernel and init it
	*/
	eError = DevmemAcquireCpuVirtAddr(psRayContext->psFWRayContextMemDesc,
									  (void **)&pFWRayContext);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,"%s: Failed to map firmware %s ray context to CPU",
								__FUNCTION__,
								PVRSRVGetErrorStringKM(eError)));
		goto fail_rscontext;
	}

	
	for (i = 0; i < DPX_MAX_RAY_CONTEXTS; i++)
	{
		/* Allocate the frame context client CCB */
		eError = RGXCreateCCB(psDevInfo,
							  RGX_RTU_CCB_SIZE_LOG2,
							  psConnection,
							  REQ_TYPE_FC0 + i,
							  psRayContext->sRSData.psServerCommonContext,
							  &psRayContext->sRSData.psFCClientCCB[i],
							  &psRayContext->sRSData.psFCClientCCBMemDesc[i],
							  &psRayContext->sRSData.psFCClientCCBCtrlMemDesc[i]);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR, "%s: failed to create CCB for frame context %u (%s)",
									__FUNCTION__,
									i,
									PVRSRVGetErrorStringKM(eError)));
			goto fail_rscontext;
		}

		/* Set the firmware CCB device addresses in the firmware common context */
		RGXSetFirmwareAddress(&pFWRayContext->psCCB[i],
							  psRayContext->sRSData.psFCClientCCBMemDesc[i],
							  0, RFW_FWADDR_FLAG_NONE);
		RGXSetFirmwareAddress(&pFWRayContext->psCCBCtl[i],
							  psRayContext->sRSData.psFCClientCCBCtrlMemDesc[i],
							  0, RFW_FWADDR_FLAG_NONE);
	}
	
	pFWRayContext->ui32ActiveFCMask = 0;
	pFWRayContext->ui32NextFC = RGXFWIF_INVALID_FRAME_CONTEXT;

	/* We've finished the setup so release the CPU mapping */
	DevmemReleaseCpuVirtAddr(psRayContext->psFWRayContextMemDesc);	
		
	/*
		As the common context alloc will dump the SH and RS common contexts
		after the've been setup we skip of the 2 common contexts and dump the
		rest of the structure
	*/
	PDUMPCOMMENT("Dump shared part of ray context context");
	DevmemPDumpLoadMem(psRayContext->psFWRayContextMemDesc,
					   (sizeof(RGXFWIF_FWCOMMONCONTEXT) * 2),
					   sizeof(RGXFWIF_FWRAYCONTEXT) - (sizeof(RGXFWIF_FWCOMMONCONTEXT) * 2),
					   PDUMP_FLAGS_CONTINUOUS);

	{
		PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;

		OSWRLockAcquireWrite(psDevInfo->hRaytraceCtxListLock);
		dllist_add_to_tail(&(psDevInfo->sRaytraceCtxtListHead), &(psRayContext->sListNode));
		OSWRLockReleaseWrite(psDevInfo->hRaytraceCtxListLock);
	}

	*ppsRayContext= psRayContext;
	return PVRSRV_OK;

fail_rscontext:
	_DestroySHContext(&psRayContext->sSHData,
					  psDeviceNode,
					  psRayContext->psCleanupSync);
fail_shcontext:
fail_frameworkcopy:
	DevmemFwFree(psDevInfo, psRayContext->psFWFrameworkMemDesc);
fail_frameworkcreate:
	SyncPrimFree(psRayContext->psCleanupSync);
fail_syncalloc:
	DevmemFwFree(psDevInfo, psRayContext->psFWRayContextMemDesc);
fail_fwraycontext:
	OSFreeMem(psRayContext);
	PVR_ASSERT(eError != PVRSRV_OK);

	return eError;
}


/*
 * PVRSRVRGXDestroyRayContextKM
 */
IMG_EXPORT
PVRSRV_ERROR PVRSRVRGXDestroyRayContextKM(RGX_SERVER_RAY_CONTEXT *psRayContext)
{
	PVRSRV_ERROR				eError;
	IMG_UINT32 i;
	PVRSRV_RGXDEV_INFO *psDevInfo = psRayContext->psDeviceNode->pvDevice;

	/* remove node from list before calling destroy - as destroy, if successful
	 * will invalidate the node
	 * must be re-added if destroy fails
	 */
	OSWRLockAcquireWrite(psDevInfo->hRaytraceCtxListLock);
	dllist_remove_node(&(psRayContext->sListNode));
	OSWRLockReleaseWrite(psDevInfo->hRaytraceCtxListLock);

	/* Cleanup the TA if we haven't already */
	if ((psRayContext->ui32CleanupStatus & RAY_CLEANUP_SH_COMPLETE) == 0)
	{
		eError = _DestroySHContext(&psRayContext->sSHData,
								   psRayContext->psDeviceNode,
								   psRayContext->psCleanupSync);
		if (eError != PVRSRV_ERROR_RETRY)
		{
			psRayContext->ui32CleanupStatus |= RAY_CLEANUP_SH_COMPLETE;
		}
		else
		{
			goto e0;
		}
	}

	/* Cleanup the RS if we haven't already */
	if ((psRayContext->ui32CleanupStatus & RAY_CLEANUP_RS_COMPLETE) == 0)
	{
		eError = _DestroyRSContext(&psRayContext->sRSData,
								   psRayContext->psDeviceNode,
								   psRayContext->psCleanupSync);
		if (eError != PVRSRV_ERROR_RETRY)
		{
			psRayContext->ui32CleanupStatus |= RAY_CLEANUP_RS_COMPLETE;
		}
		else
		{
			goto e0;
		}
	}

#if 0
	/*
	 * 	FIXME - De-allocate RPM freelists (should be called from UM)
	 */
	RGXDestroyRPMFreeList(psRayContext->sSHData.psSHFFreeList);
	RGXDestroyRPMFreeList(psRayContext->sSHData.psSHGFreeList);
#endif
	
	for (i = 0; i < DPX_MAX_RAY_CONTEXTS; i++)
	{
		RGXUnsetFirmwareAddress(psRayContext->sRSData.psFCClientCCBMemDesc[i]);
		RGXUnsetFirmwareAddress(psRayContext->sRSData.psFCClientCCBCtrlMemDesc[i]);
		RGXDestroyCCB(psDevInfo, psRayContext->sRSData.psFCClientCCB[i]);
	}

	/*
		Only if both TA and 3D contexts have been cleaned up can we
		free the shared resources
	*/
	if (psRayContext->ui32CleanupStatus == (RAY_CLEANUP_RS_COMPLETE | RAY_CLEANUP_SH_COMPLETE))
	{
		/* Free the framework buffer */
		DevmemFwFree(psDevInfo, psRayContext->psFWFrameworkMemDesc);
	
		/* Free the firmware ray context */
		DevmemFwFree(psDevInfo, psRayContext->psFWRayContextMemDesc);

		/* Free the cleanup sync */
		SyncPrimFree(psRayContext->psCleanupSync);

		OSFreeMem(psRayContext);
	}

	return PVRSRV_OK;

e0:
	OSWRLockAcquireWrite(psDevInfo->hRaytraceCtxListLock);
	dllist_add_to_tail(&(psDevInfo->sRaytraceCtxtListHead), &(psRayContext->sListNode));
	OSWRLockReleaseWrite(psDevInfo->hRaytraceCtxListLock);
	return eError;
}

/*
 * PVRSRVRGXKickRSKM
 */
IMG_EXPORT
PVRSRV_ERROR PVRSRVRGXKickRSKM(RGX_SERVER_RAY_CONTEXT		*psRayContext,
								IMG_UINT32					ui32ClientCacheOpSeqNum,
								IMG_UINT32					ui32ClientFenceCount,
								SYNC_PRIMITIVE_BLOCK			**pauiClientFenceUFOSyncPrimBlock,
								IMG_UINT32					*paui32ClientFenceSyncOffset,
								IMG_UINT32					*paui32ClientFenceValue,
								IMG_UINT32					ui32ClientUpdateCount,
								SYNC_PRIMITIVE_BLOCK			**pauiClientUpdateUFOSyncPrimBlock,
								IMG_UINT32					*paui32ClientUpdateSyncOffset,
								IMG_UINT32					*paui32ClientUpdateValue,
								IMG_UINT32					ui32ServerSyncPrims,
								IMG_UINT32					*paui32ServerSyncFlags,
								SERVER_SYNC_PRIMITIVE 		**pasServerSyncs,
								IMG_UINT32					ui32CmdSize,
								IMG_PBYTE					pui8DMCmd,
								IMG_UINT32					ui32FCCmdSize,
								IMG_PBYTE					pui8FCDMCmd,
								IMG_UINT32					ui32FrameContextID,
								IMG_UINT32					ui32PDumpFlags,
								IMG_UINT32					ui32ExtJobRef)
{
	RGXFWIF_KCCB_CMD		sRSKCCBCmd;
	RGX_CCB_CMD_HELPER_DATA	asRSCmdHelperData[1] = {{0}};
	RGX_CCB_CMD_HELPER_DATA asFCCmdHelperData[1] = {{0}};
	PVRSRV_ERROR			eError;
	PVRSRV_ERROR			eError1;
	PVRSRV_ERROR			eError2;
	RGX_SERVER_RAY_RS_DATA *psRSData = &psRayContext->sRSData;
	IMG_UINT32				i;
	IMG_UINT32				ui32FCWoff;
	IMG_UINT32				ui32RTUCmdOffset = 0;
	IMG_UINT32				ui32JobId;
	IMG_UINT32				ui32FWCtx;

	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
	
	ui32JobId = OSAtomicIncrement(&psRayContext->hJobId);

	eError = SyncAddrListPopulate(&psRayContext->sSyncAddrListFence,
							ui32ClientFenceCount,
							pauiClientFenceUFOSyncPrimBlock,
							paui32ClientFenceSyncOffset);
	if(eError != PVRSRV_OK)
	{
		goto err_populate_sync_addr_list;
	}

	eError = SyncAddrListPopulate(&psRayContext->sSyncAddrListUpdate,
							ui32ClientUpdateCount,
							pauiClientUpdateUFOSyncPrimBlock,
							paui32ClientUpdateSyncOffset);
	if(eError != PVRSRV_OK)
	{
		goto err_populate_sync_addr_list;
	}

	/* Sanity check the server fences */
	for (i=0;i<ui32ServerSyncPrims;i++)
	{
		if (!(paui32ServerSyncFlags[i] & PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK))
		{
			PVR_DPF((PVR_DBG_ERROR, "%s: Server fence (on RS) must fence", __FUNCTION__));
			return PVRSRV_ERROR_INVALID_SYNC_PRIM_OP;
		}
	}

	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRayContext->psDeviceNode->pvDevice,
	                          & pPreAddr,
	                          & pPostAddr,
	                          & pRMWUFOAddr);


    if(pui8DMCmd != NULL)
	{
		eError = RGXCmdHelperInitCmdCCB(psRSData->psFCClientCCB[ui32FrameContextID],
	                                0,
                                    NULL,
                                    NULL,
	                                ui32ClientUpdateCount,
	                                psRayContext->sSyncAddrListUpdate.pasFWAddrs,
	                                paui32ClientUpdateValue,
	                                ui32ServerSyncPrims,
	                                paui32ServerSyncFlags,
	                                SYNC_FLAG_MASK_ALL,
	                                pasServerSyncs,
	                                ui32CmdSize,
	                                pui8DMCmd,
	                                & pPreAddr,
	                                & pPostAddr,
	                                & pRMWUFOAddr,
	                                RGXFWIF_CCB_CMD_TYPE_RTU,
	                                ui32ExtJobRef,
	                                ui32JobId,
	                                ui32PDumpFlags,
	                                NULL,
	                                "FC",
	                                asFCCmdHelperData);
	}
	else
	{
		eError = RGXCmdHelperInitCmdCCB(psRSData->psFCClientCCB[ui32FrameContextID],
	                                0,
                                    NULL,
                                    NULL,
	                                ui32ClientUpdateCount,
	                                psRayContext->sSyncAddrListUpdate.pasFWAddrs,
	                                paui32ClientUpdateValue,
	                                ui32ServerSyncPrims,
	                                paui32ServerSyncFlags,
	                                SYNC_FLAG_MASK_ALL,
	                                pasServerSyncs,
	                                ui32CmdSize,
	                                pui8DMCmd,
	                                & pPreAddr,
	                                & pPostAddr,
	                                & pRMWUFOAddr,
	                                RGXFWIF_CCB_CMD_TYPE_NULL,
	                                ui32ExtJobRef,
	                                ui32JobId,
	                                ui32PDumpFlags,
	                                NULL,
	                                "FC",
	                                asFCCmdHelperData);

	}

	if (eError != PVRSRV_OK)
	{
		goto PVRSRVRGXKickRSKM_Exit;
	}

	eError = RGXCmdHelperAcquireCmdCCB(IMG_ARR_NUM_ELEMS(asFCCmdHelperData),
	                                   asFCCmdHelperData);
	if (eError != PVRSRV_OK)
	{
		goto PVRSRVRGXKickRSKM_Exit;
	}
	
	ui32FCWoff = RGXCmdHelperGetCommandSize(IMG_ARR_NUM_ELEMS(asFCCmdHelperData),
	                                        asFCCmdHelperData);
	
	*(IMG_UINT32*)pui8FCDMCmd = RGXGetHostWriteOffsetCCB(psRSData->psFCClientCCB[ui32FrameContextID]) + ui32FCWoff;

	/*
		We should reserved space in the kernel CCB here and fill in the command
		directly.
		This is so if there isn't space in the kernel CCB we can return with
		retry back to services client before we take any operations
	*/

	/*
		We might only be kicking for flush out a padding packet so only submit
		the command if the create was successful
	*/
	eError1 = RGXCmdHelperInitCmdCCB(FWCommonContextGetClientCCB(psRSData->psServerCommonContext),
	                                 ui32ClientFenceCount,
	                                 psRayContext->sSyncAddrListFence.pasFWAddrs,
	                                 paui32ClientFenceValue,
	                                 0,
                                     NULL,
                                     NULL,
	                                 ui32ServerSyncPrims,
	                                 paui32ServerSyncFlags,
	                                 SYNC_FLAG_MASK_ALL,
	                                 pasServerSyncs,
	                                 ui32FCCmdSize,
	                                 pui8FCDMCmd,
                                     NULL,
	                                 & pPostAddr,
	                                 & pRMWUFOAddr,
	                                 RGXFWIF_CCB_CMD_TYPE_RTU_FC,
	                                 ui32ExtJobRef,
	                                 ui32JobId,
	                                 ui32PDumpFlags,
	                                 NULL,
	                                 "RS",
	                                 asRSCmdHelperData);
	if (eError1 != PVRSRV_OK)
	{
		goto PVRSRVRGXKickRSKM_Exit;
	}

	eError1 = RGXCmdHelperAcquireCmdCCB(IMG_ARR_NUM_ELEMS(asRSCmdHelperData),
	                                    asRSCmdHelperData);
	if (eError1 != PVRSRV_OK)
	{
		goto PVRSRVRGXKickRSKM_Exit;
	}
	
	
	/*
		We should reserved space in the kernel CCB here and fill in the command
		directly.
		This is so if there isn't space in the kernel CCB we can return with
		retry back to services client before we take any operations
	*/

	/*
		We might only be kicking for flush out a padding packet so only submit
		the command if the create was successful
	*/
	if (eError == PVRSRV_OK)
	{
		/*
			All the required resources are ready at this point, we can't fail so
			take the required server sync operations and commit all the resources
		*/
		RGXCmdHelperReleaseCmdCCB(IMG_ARR_NUM_ELEMS(asFCCmdHelperData),
		                          asFCCmdHelperData, "FC", 0);
	}
		
	if (eError1 == PVRSRV_OK)
	{
		/*
			All the required resources are ready at this point, we can't fail so
			take the required server sync operations and commit all the resources
		*/
		ui32RTUCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRSData->psServerCommonContext));
		RGXCmdHelperReleaseCmdCCB(IMG_ARR_NUM_ELEMS(asRSCmdHelperData),
		                          asRSCmdHelperData, "RS",
		                          FWCommonContextGetFWAddress(psRSData->psServerCommonContext).ui32Addr);
	}
	
	/*
	 * Construct the kernel RTU CCB command.
	 * (Safe to release reference to ray context virtual address because
	 * ray context destruction must flush the firmware).
	 */
	sRSKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
	sRSKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psRSData->psServerCommonContext);
	sRSKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRSData->psServerCommonContext));
	sRSKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;

	ui32FWCtx = FWCommonContextGetFWAddress(psRSData->psServerCommonContext).ui32Addr;

	HTBLOGK(HTB_SF_MAIN_KICK_RTU,
			sRSKCCBCmd.uCmdData.sCmdKickData.psContext,
			ui32RTUCmdOffset
			);
	RGX_HWPERF_HOST_ENQ(psRayContext, OSGetCurrentClientProcessIDKM(),
	                    ui32FWCtx, ui32ExtJobRef, ui32JobId,
	                    RGX_HWPERF_KICK_TYPE_RS);

	/*
	 * Submit the RTU command to the firmware.
	 */
	LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
	{
		eError2 = RGXScheduleCommand(psRayContext->psDeviceNode->pvDevice,
									RGXFWIF_DM_RTU,
									&sRSKCCBCmd,
									sizeof(sRSKCCBCmd),
									ui32ClientCacheOpSeqNum,
									ui32PDumpFlags);
		if (eError2 != PVRSRV_ERROR_RETRY)
		{
			break;
		}
		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
	} END_LOOP_UNTIL_TIMEOUT();

	if (eError2 != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKickRSKM failed to schedule kernel RTU command. Error:%u", eError));
		if (eError == PVRSRV_OK)
		{
			eError = eError2;
		}
		goto PVRSRVRGXKickRSKM_Exit;
	}
	else
	{
#if defined(SUPPORT_GPUTRACE_EVENTS)
		RGXHWPerfFTraceGPUEnqueueEvent(psRayContext->psDeviceNode->pvDevice,
				ui32FWCtx, ui32JobId, RGX_HWPERF_KICK_TYPE_RS);
#endif
	}


PVRSRVRGXKickRSKM_Exit:
err_populate_sync_addr_list:
	return eError;
}

/*
 * PVRSRVRGXKickVRDMKM
 */
IMG_EXPORT
PVRSRV_ERROR PVRSRVRGXKickVRDMKM(RGX_SERVER_RAY_CONTEXT		*psRayContext,
								 IMG_UINT32					ui32ClientCacheOpSeqNum,
								 IMG_UINT32					ui32ClientFenceCount,
								 SYNC_PRIMITIVE_BLOCK			**pauiClientFenceUFOSyncPrimBlock,
								 IMG_UINT32					*paui32ClientFenceSyncOffset,
								 IMG_UINT32					*paui32ClientFenceValue,
								 IMG_UINT32					ui32ClientUpdateCount,
								 SYNC_PRIMITIVE_BLOCK			**pauiClientUpdateUFOSyncPrimBlock,
								 IMG_UINT32					*paui32ClientUpdateSyncOffset,
								 IMG_UINT32					*paui32ClientUpdateValue,
								 IMG_UINT32					ui32ServerSyncPrims,
								 IMG_UINT32					*paui32ServerSyncFlags,
								 SERVER_SYNC_PRIMITIVE 		**pasServerSyncs,
								 IMG_UINT32					ui32CmdSize,
								 IMG_PBYTE					pui8DMCmd,
								 IMG_UINT32					ui32PDumpFlags,
								 IMG_UINT32					ui32ExtJobRef)
{
	RGXFWIF_KCCB_CMD		sSHKCCBCmd;
	RGX_CCB_CMD_HELPER_DATA	sCmdHelperData;
	PVRSRV_ERROR			eError;
	PVRSRV_ERROR			eError2;
	RGX_SERVER_RAY_SH_DATA *psSHData = &psRayContext->sSHData;
	IMG_UINT32				i;
	IMG_UINT32				ui32SHGCmdOffset = 0;
	IMG_UINT32				ui32JobId;
	IMG_UINT32				ui32FWCtx;

	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;

	ui32JobId = OSAtomicIncrement(&psRayContext->hJobId);

	eError = SyncAddrListPopulate(&psRayContext->sSyncAddrListFence,
							ui32ClientFenceCount,
							pauiClientFenceUFOSyncPrimBlock,
							paui32ClientFenceSyncOffset);
	if(eError != PVRSRV_OK)
	{
		goto err_populate_sync_addr_list;
	}

	eError = SyncAddrListPopulate(&psRayContext->sSyncAddrListUpdate,
							ui32ClientUpdateCount,
							pauiClientUpdateUFOSyncPrimBlock,
							paui32ClientUpdateSyncOffset);
	if(eError != PVRSRV_OK)
	{
		goto err_populate_sync_addr_list;
	}

	/* Sanity check the server fences */
	for (i=0;i<ui32ServerSyncPrims;i++)
	{
		if (!(paui32ServerSyncFlags[i] & PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK))
		{
			PVR_DPF((PVR_DBG_ERROR, "%s: Server fence (on SH) must fence", __FUNCTION__));
			return PVRSRV_ERROR_INVALID_SYNC_PRIM_OP;
		}
	}

	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRayContext->psDeviceNode->pvDevice,
	                          & pPreAddr,
	                          & pPostAddr,
	                          & pRMWUFOAddr);

	eError = RGXCmdHelperInitCmdCCB(FWCommonContextGetClientCCB(psSHData->psServerCommonContext),
	                                ui32ClientFenceCount,
	                                psRayContext->sSyncAddrListFence.pasFWAddrs,
	                                paui32ClientFenceValue,
	                                ui32ClientUpdateCount,
	                                psRayContext->sSyncAddrListUpdate.pasFWAddrs,
	                                paui32ClientUpdateValue,
	                                ui32ServerSyncPrims,
	                                paui32ServerSyncFlags,
	                                SYNC_FLAG_MASK_ALL,
	                                pasServerSyncs,
	                                ui32CmdSize,
	                                pui8DMCmd,
	                                & pPreAddr,
	                                & pPostAddr,
	                                & pRMWUFOAddr,
	                                RGXFWIF_CCB_CMD_TYPE_SHG,
	                                ui32ExtJobRef,
	                                ui32JobId,
	                                ui32PDumpFlags,
	                                NULL,
	                                "SH",
	                                &sCmdHelperData);

	if (eError != PVRSRV_OK)
	{
		goto PVRSRVRGXKickSHKM_Exit;
	}

	eError = RGXCmdHelperAcquireCmdCCB(1, &sCmdHelperData);
	if (eError != PVRSRV_OK)
	{
		goto PVRSRVRGXKickSHKM_Exit;
	}
	
	
	/*
		We should reserve space in the kernel CCB here and fill in the command
		directly.
		This is so if there isn't space in the kernel CCB we can return with
		retry back to services client before we take any operations
	*/

	/*
		We might only be kicking for flush out a padding packet so only submit
		the command if the create was successful
	*/
	if (eError == PVRSRV_OK)
	{
		/*
			All the required resources are ready at this point, we can't fail so
			take the required server sync operations and commit all the resources
		*/
		ui32SHGCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psSHData->psServerCommonContext));
		RGXCmdHelperReleaseCmdCCB(1, &sCmdHelperData, "SH", FWCommonContextGetFWAddress(psSHData->psServerCommonContext).ui32Addr);
	}
	
	/*
	 * Construct the kernel SHG CCB command.
	 * (Safe to release reference to ray context virtual address because
	 * ray context destruction must flush the firmware).
	 */
	sSHKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
	sSHKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psSHData->psServerCommonContext);
	sSHKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psSHData->psServerCommonContext));
	sSHKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;

	ui32FWCtx = FWCommonContextGetFWAddress(psSHData->psServerCommonContext).ui32Addr;

	HTBLOGK(HTB_SF_MAIN_KICK_SHG,
			sSHKCCBCmd.uCmdData.sCmdKickData.psContext,
			ui32SHGCmdOffset
			);
	RGX_HWPERF_HOST_ENQ(psRayContext, OSGetCurrentClientProcessIDKM(),
	                    ui32FWCtx, ui32ExtJobRef, ui32JobId,
	                    RGX_HWPERF_KICK_TYPE_VRDM);

	/*
	 * Submit the RTU command to the firmware.
	 */
	LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
	{
		eError2 = RGXScheduleCommand(psRayContext->psDeviceNode->pvDevice,
									RGXFWIF_DM_SHG,
									&sSHKCCBCmd,
									sizeof(sSHKCCBCmd),
									ui32ClientCacheOpSeqNum,
									ui32PDumpFlags);
		if (eError2 != PVRSRV_ERROR_RETRY)
		{
			break;
		}
		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
	} END_LOOP_UNTIL_TIMEOUT();

	if (eError2 != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKickSHKM failed to schedule kernel RTU command. Error:%u", eError));
		if (eError == PVRSRV_OK)
		{
			eError = eError2;
		}
		goto PVRSRVRGXKickSHKM_Exit;
	}
	else
	{
#if defined(SUPPORT_GPUTRACE_EVENTS)
		RGXHWPerfFTraceGPUEnqueueEvent(psRayContext->psDeviceNode->pvDevice,
				ui32FWCtx, ui32JobId, RGX_HWPERF_KICK_TYPE_VRDM);
#endif
	}


PVRSRVRGXKickSHKM_Exit:
err_populate_sync_addr_list:
	return eError;
}

PVRSRV_ERROR PVRSRVRGXSetRayContextPriorityKM(CONNECTION_DATA *psConnection,
                                              PVRSRV_DEVICE_NODE * psDeviceNode,
												 RGX_SERVER_RAY_CONTEXT *psRayContext,
												 IMG_UINT32 ui32Priority)
{
	PVRSRV_ERROR eError;

	PVR_UNREFERENCED_PARAMETER(psDeviceNode);

	if (psRayContext->sSHData.ui32Priority != ui32Priority)
	{
		eError = ContextSetPriority(psRayContext->sSHData.psServerCommonContext,
									psConnection,
									psRayContext->psDeviceNode->pvDevice,
									ui32Priority,
									RGXFWIF_DM_SHG);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the SH part of the rendercontext (%s)", __FUNCTION__, PVRSRVGetErrorStringKM(eError)));
			goto fail_shcontext;
		}

		psRayContext->sSHData.ui32Priority = ui32Priority;
	}

	if (psRayContext->sRSData.ui32Priority != ui32Priority)
	{
		eError = ContextSetPriority(psRayContext->sRSData.psServerCommonContext,
									psConnection,
									psRayContext->psDeviceNode->pvDevice,
									ui32Priority,
									RGXFWIF_DM_RTU);
		if (eError != PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the RS part of the rendercontext (%s)", __FUNCTION__, PVRSRVGetErrorStringKM(eError)));
			goto fail_rscontext;
		}

		psRayContext->sRSData.ui32Priority = ui32Priority;
	}
	return PVRSRV_OK;

fail_rscontext:
fail_shcontext:
	PVR_ASSERT(eError != PVRSRV_OK);
	return eError;
}

void CheckForStalledRayCtxt(PVRSRV_RGXDEV_INFO *psDevInfo,
				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
				void *pvDumpDebugFile)
{
	DLLIST_NODE *psNode, *psNext;
	OSWRLockAcquireRead(psDevInfo->hRaytraceCtxListLock);
	dllist_foreach_node(&psDevInfo->sRaytraceCtxtListHead, psNode, psNext)
	{
		RGX_SERVER_RAY_CONTEXT *psCurrentServerRayCtx =
			IMG_CONTAINER_OF(psNode, RGX_SERVER_RAY_CONTEXT, sListNode);

		DumpStalledFWCommonContext(psCurrentServerRayCtx->sSHData.psServerCommonContext,
								   pfnDumpDebugPrintf, pvDumpDebugFile);
		DumpStalledFWCommonContext(psCurrentServerRayCtx->sRSData.psServerCommonContext,
								   pfnDumpDebugPrintf, pvDumpDebugFile);
	}
	OSWRLockReleaseRead(psDevInfo->hRaytraceCtxListLock);
}

IMG_UINT32 CheckForStalledClientRayCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
{
	DLLIST_NODE *psNode, *psNext;
	IMG_UINT32 ui32ContextBitMask = 0;

	OSWRLockAcquireRead(psDevInfo->hRaytraceCtxListLock);

	dllist_foreach_node(&psDevInfo->sRaytraceCtxtListHead, psNode, psNext)
	{
		RGX_SERVER_RAY_CONTEXT *psCurrentServerRayCtx =
			IMG_CONTAINER_OF(psNode, RGX_SERVER_RAY_CONTEXT, sListNode);
		if(NULL != psCurrentServerRayCtx->sSHData.psServerCommonContext)
		{
			if (CheckStalledClientCommonContext(psCurrentServerRayCtx->sSHData.psServerCommonContext, RGX_KICK_TYPE_DM_RTU) == PVRSRV_ERROR_CCCB_STALLED)
			{
				ui32ContextBitMask |= RGX_KICK_TYPE_DM_RTU;
			}
		}

		if(NULL != psCurrentServerRayCtx->sRSData.psServerCommonContext)
		{
			if (CheckStalledClientCommonContext(psCurrentServerRayCtx->sRSData.psServerCommonContext, RGX_KICK_TYPE_DM_SHG) == PVRSRV_ERROR_CCCB_STALLED)
			{
				ui32ContextBitMask |= RGX_KICK_TYPE_DM_SHG;
			}
		}
	}

	OSWRLockReleaseRead(psDevInfo->hRaytraceCtxListLock);
	return ui32ContextBitMask;
}

/******************************************************************************
 End of file (rgxSHGRTU.c)
******************************************************************************/