asgardius/android_kernel_samsung_heroqlte
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
android_kernel_samsung_hero.../drivers/gpu/msm/adreno_ringbuffer.c
HackerOO7 2bd1731e0a init: OSRC G9300ZCU2APF7
2016-08-17 16:41:52 +08:00

1359 lines
39 KiB
C
Raw Blame History

/* Copyright (c) 2002,2007-2015, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
* only version 2 as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
*/
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/log2.h>
#include <linux/time.h>
#include <linux/delay.h>
#include "kgsl.h"
#include "kgsl_sharedmem.h"
#include "kgsl_cffdump.h"
#include "kgsl_trace.h"
#include "kgsl_pwrctrl.h"
#include "adreno.h"
#include "adreno_pm4types.h"
#include "adreno_ringbuffer.h"
#include "a3xx_reg.h"
#include "adreno_a5xx.h"
#define GSL_RB_NOP_SIZEDWORDS 2
#define ADRENO_RB_PREEMPT_TOKEN_IB_DWORDS 50
#define ADRENO_RB_PREEMPT_TOKEN_DWORDS 125
#define RB_HOSTPTR(_rb, _pos) \
((unsigned int *) ((_rb)->buffer_desc.hostptr + \
((_pos) * sizeof(unsigned int))))
#define RB_GPUADDR(_rb, _pos) \
((_rb)->buffer_desc.gpuaddr + ((_pos) * sizeof(unsigned int)))
static void _cff_write_ringbuffer(struct adreno_ringbuffer *rb)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
struct kgsl_device *device = &adreno_dev->dev;
uint64_t gpuaddr;
unsigned int *hostptr;
size_t size;
if (device->cff_dump_enable == 0)
return;
/*
* This code is predicated on the fact that we write a full block of
* stuff without wrapping
*/
BUG_ON(rb->wptr < rb->last_wptr);
size = (rb->wptr - rb->last_wptr) * sizeof(unsigned int);
hostptr = RB_HOSTPTR(rb, rb->last_wptr);
gpuaddr = RB_GPUADDR(rb, rb->last_wptr);
kgsl_cffdump_memcpy(device, gpuaddr, hostptr, size);
}
void adreno_ringbuffer_submit(struct adreno_ringbuffer *rb,
struct adreno_submit_time *time)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
BUG_ON(rb->wptr == 0);
/* Write the changes to CFF if so enabled */
_cff_write_ringbuffer(rb);
/*
* Read the current GPU ticks and wallclock for most accurate
* profiling
*/
if (time != NULL) {
/*
* Here we are attempting to create a mapping between the
* GPU time domain (alwayson counter) and the CPU time domain
* (local_clock) by sampling both values as close together as
* possible. This is useful for many types of debugging and
* profiling. In order to make this mapping as accurate as
* possible, we must turn off interrupts to avoid running
* interrupt handlers between the two samples.
*/
unsigned long flags;
local_irq_save(flags);
/* Read always on registers */
if (!adreno_is_a3xx(adreno_dev)) {
adreno_readreg64(adreno_dev,
ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO,
ADRENO_REG_RBBM_ALWAYSON_COUNTER_HI,
&time->ticks);
/*
* Mask hi bits as they may be incorrect on
* a4x and some a5x
*/
if (ADRENO_GPUREV(adreno_dev) >= 400 &&
ADRENO_GPUREV(adreno_dev) <= ADRENO_REV_A530)
time->ticks &= 0xFFFFFFFF;
}
else
time->ticks = 0;
/* Get the kernel clock for time since boot */
time->ktime = local_clock();
/* Get the timeofday for the wall time (for the user) */
getnstimeofday(&time->utime);
local_irq_restore(flags);
}
/* Memory barrier before informing the hardware of new commands */
mb();
if (adreno_preempt_state(adreno_dev, ADRENO_DISPATCHER_PREEMPT_CLEAR) &&
(adreno_dev->cur_rb == rb)) {
/*
* Let the pwrscale policy know that new commands have
* been submitted.
*/
kgsl_pwrscale_busy(rb->device);
adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, rb->wptr);
}
}
static int
adreno_ringbuffer_waitspace(struct adreno_ringbuffer *rb,
unsigned int numcmds, int wptr_ahead)
{
int nopcount = 0;
unsigned int freecmds;
unsigned int wptr = rb->wptr;
unsigned int *cmds = NULL;
uint64_t gpuaddr;
unsigned long wait_time;
unsigned long wait_timeout = msecs_to_jiffies(ADRENO_IDLE_TIMEOUT);
unsigned int rptr;
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
/* if wptr ahead, fill the remaining with NOPs */
if (wptr_ahead) {
/* -1 for header */
nopcount = KGSL_RB_DWORDS - rb->wptr - 1;
cmds = RB_HOSTPTR(rb, rb->wptr);
gpuaddr = RB_GPUADDR(rb, rb->wptr);
rptr = adreno_get_rptr(rb);
/* For non current rb we don't expect the rptr to move */
if ((adreno_dev->cur_rb != rb ||
!adreno_preempt_state(adreno_dev,
ADRENO_DISPATCHER_PREEMPT_CLEAR)) &&
!rptr)
return -ENOSPC;
/* Make sure that rptr is not 0 before submitting
* commands at the end of ringbuffer. We do not
* want the rptr and wptr to become equal when
* the ringbuffer is not empty */
wait_time = jiffies + wait_timeout;
while (!rptr) {
rptr = adreno_get_rptr(rb);
if (time_after(jiffies, wait_time))
return -ETIMEDOUT;
}
rb->wptr = 0;
}
rptr = adreno_get_rptr(rb);
freecmds = rptr - rb->wptr;
if (freecmds == 0 || freecmds > numcmds)
goto done;
/* non current rptr will not advance anyway or if preemption underway */
if (adreno_dev->cur_rb != rb ||
!adreno_preempt_state(adreno_dev,
ADRENO_DISPATCHER_PREEMPT_CLEAR)) {
rb->wptr = wptr;
return -ENOSPC;
}
wait_time = jiffies + wait_timeout;
/* wait for space in ringbuffer */
while (1) {
rptr = adreno_get_rptr(rb);
freecmds = rptr - rb->wptr;
if (freecmds == 0 || freecmds > numcmds)
break;
if (time_after(jiffies, wait_time)) {
KGSL_DRV_ERR(rb->device,
"Timed out waiting for freespace in RB rptr: 0x%x, wptr: 0x%x, rb id %d\n",
rptr, wptr, rb->id);
return -ETIMEDOUT;
}
}
done:
if (wptr_ahead) {
*cmds = cp_packet(adreno_dev, CP_NOP, nopcount);
kgsl_cffdump_write(rb->device, gpuaddr, *cmds);
}
return 0;
}
unsigned int *adreno_ringbuffer_allocspace(struct adreno_ringbuffer *rb,
unsigned int numcmds)
{
unsigned int *ptr = NULL;
int ret = 0;
unsigned int rptr;
BUG_ON(numcmds >= KGSL_RB_DWORDS);
rptr = adreno_get_rptr(rb);
/* check for available space */
if (rb->wptr >= rptr) {
/* wptr ahead or equal to rptr */
/* reserve dwords for nop packet */
if ((rb->wptr + numcmds) > (KGSL_RB_DWORDS -
GSL_RB_NOP_SIZEDWORDS))
ret = adreno_ringbuffer_waitspace(rb, numcmds, 1);
} else {
/* wptr behind rptr */
if ((rb->wptr + numcmds) >= rptr)
ret = adreno_ringbuffer_waitspace(rb, numcmds, 0);
/* check for remaining space */
/* reserve dwords for nop packet */
if (!ret && (rb->wptr + numcmds) > (KGSL_RB_DWORDS -
GSL_RB_NOP_SIZEDWORDS))
ret = adreno_ringbuffer_waitspace(rb, numcmds, 1);
}
if (!ret) {
rb->last_wptr = rb->wptr;
ptr = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
rb->wptr += numcmds;
} else
ptr = ERR_PTR(ret);
return ptr;
}
/**
* _ringbuffer_setup_common() - Ringbuffer start
* @rb: Pointer to adreno ringbuffer
*
* Setup ringbuffer for GPU.
*/
static void _ringbuffer_setup_common(struct adreno_ringbuffer *rb)
{
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_ringbuffer *rb_temp;
int i;
FOR_EACH_RINGBUFFER(adreno_dev, rb_temp, i) {
kgsl_sharedmem_set(rb_temp->device,
&(rb_temp->buffer_desc), 0,
0xAA, KGSL_RB_SIZE);
rb_temp->wptr = 0;
rb_temp->rptr = 0;
rb_temp->wptr_preempt_end = 0xFFFFFFFF;
rb_temp->starve_timer_state =
ADRENO_DISPATCHER_RB_STARVE_TIMER_UNINIT;
adreno_iommu_set_pt_generate_rb_cmds(rb_temp,
device->mmu.defaultpagetable);
}
/*
* The size of the ringbuffer in the hardware is the log2
* representation of the size in quadwords (sizedwords / 2).
* Also disable the host RPTR shadow register as it might be unreliable
* in certain circumstances.
*/
adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_CNTL,
(ilog2(KGSL_RB_DWORDS >> 1) & 0x3F) |
(1 << 27));
adreno_writereg64(adreno_dev, ADRENO_REG_CP_RB_BASE,
ADRENO_REG_CP_RB_BASE_HI, rb->buffer_desc.gpuaddr);
/* CP ROQ queue sizes (bytes) - RB:16, ST:16, IB1:32, IB2:64 */
if (adreno_is_a305(adreno_dev) || adreno_is_a305c(adreno_dev) ||
adreno_is_a306(adreno_dev) || adreno_is_a320(adreno_dev) ||
adreno_is_a304(adreno_dev))
kgsl_regwrite(device, A3XX_CP_QUEUE_THRESHOLDS, 0x000E0602);
else if (adreno_is_a330(adreno_dev) || adreno_is_a305b(adreno_dev) ||
adreno_is_a310(adreno_dev))
kgsl_regwrite(device, A3XX_CP_QUEUE_THRESHOLDS, 0x003E2008);
}
/**
* _ringbuffer_start_common() - Ringbuffer start
* @rb: Pointer to adreno ringbuffer
*
* Start ringbuffer for GPU.
*/
static int _ringbuffer_start_common(struct adreno_ringbuffer *rb)
{
int status;
struct kgsl_device *device = rb->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
/* clear ME_HALT to start micro engine */
adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, 0);
/* ME init is GPU specific, so jump into the sub-function */
status = gpudev->rb_init(adreno_dev, rb);
if (status)
return status;
/* idle device to validate ME INIT */
status = adreno_spin_idle(device);
if (status) {
KGSL_DRV_ERR(rb->device,
"ringbuffer initialization failed to idle\n");
kgsl_device_snapshot(device, NULL);
}
if (gpudev->switch_to_unsecure_mode) {
status = gpudev->switch_to_unsecure_mode(adreno_dev, rb);
if (status)
return status;
status = adreno_spin_idle(device);
if (status)
KGSL_DRV_ERR(rb->device,
"switching to unsecure mode failed to idle\n");
}
return status;
}
/**
* adreno_ringbuffer_start() - Ringbuffer start
* @adreno_dev: Pointer to adreno device
* @start_type: Warm or cold start
*/
int adreno_ringbuffer_start(struct adreno_device *adreno_dev,
unsigned int start_type)
{
int status;
struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev);
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
_ringbuffer_setup_common(rb);
status = gpudev->microcode_load(adreno_dev, start_type);
if (status)
return status;
return _ringbuffer_start_common(rb);
}
void adreno_ringbuffer_stop(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = &adreno_dev->dev;
struct adreno_ringbuffer *rb;
int i;
FOR_EACH_RINGBUFFER(adreno_dev, rb, i)
kgsl_cancel_events(device, &(rb->events));
}
static int _adreno_ringbuffer_init(struct adreno_device *adreno_dev,
struct adreno_ringbuffer *rb, int id)
{
int ret;
char name[64];
rb->device = &adreno_dev->dev;
rb->id = id;
snprintf(name, sizeof(name), "rb_events-%d", id);
kgsl_add_event_group(&rb->events, NULL, name,
adreno_rb_readtimestamp, rb);
rb->timestamp = 0;
init_waitqueue_head(&rb->ts_expire_waitq);
/*
* Allocate mem for storing RB pagetables and commands to
* switch pagetable
*/
ret = kgsl_allocate_global(&adreno_dev->dev, &rb->pagetable_desc,
PAGE_SIZE, 0, KGSL_MEMDESC_PRIVILEGED);
if (ret)
return ret;
ret = kgsl_allocate_global(&adreno_dev->dev, &rb->buffer_desc,
KGSL_RB_SIZE, KGSL_MEMFLAGS_GPUREADONLY, 0);
return ret;
}
int adreno_ringbuffer_init(struct kgsl_device *device)
{
int status = 0;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
struct adreno_ringbuffer *rb;
int i;
if (ADRENO_FEATURE(adreno_dev, ADRENO_PREEMPTION))
adreno_dev->num_ringbuffers = gpudev->num_prio_levels;
else
adreno_dev->num_ringbuffers = 1;
FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
status = _adreno_ringbuffer_init(adreno_dev, rb, i);
if (status)
break;
}
if (status)
adreno_ringbuffer_close(adreno_dev);
else
adreno_dev->cur_rb = &(adreno_dev->ringbuffers[0]);
return status;
}
static void _adreno_ringbuffer_close(struct adreno_ringbuffer *rb)
{
kgsl_free_global(&rb->pagetable_desc);
kgsl_free_global(&rb->preemption_desc);
memset(&rb->pt_update_desc, 0, sizeof(struct kgsl_memdesc));
kgsl_free_global(&rb->buffer_desc);
kgsl_del_event_group(&rb->events);
memset(rb, 0, sizeof(struct adreno_ringbuffer));
}
void adreno_ringbuffer_close(struct adreno_device *adreno_dev)
{
struct adreno_ringbuffer *rb;
int i;
FOR_EACH_RINGBUFFER(adreno_dev, rb, i)
_adreno_ringbuffer_close(rb);
}
/*
* cp_secure_mode() - Put GPU in trusted mode
* @adreno_dev: Pointer to adreno device
* @cmds: Pointer to cmds to be put in the ringbuffer
* @set: 1 - secure mode, 0 - unsecure mode
*
* Add commands to the ringbuffer to put the GPU in secure mode
* or unsecure mode based on the variable set.
*/
int cp_secure_mode(struct adreno_device *adreno_dev, uint *cmds,
int set)
{
uint *start = cmds;
if (adreno_is_a4xx(adreno_dev)) {
cmds += cp_wait_for_idle(adreno_dev, cmds);
/*
* The two commands will stall the PFP until the PFP-ME-AHB
* is drained and the GPU is idle. As soon as this happens,
* the PFP will start moving again.
*/
cmds += cp_wait_for_me(adreno_dev, cmds);
/*
* Below commands are processed by ME. GPU will be
* idle when they are processed. But the PFP will continue
* to fetch instructions at the same time.
*/
*cmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
*cmds++ = 0;
*cmds++ = cp_packet(adreno_dev, CP_WIDE_REG_WRITE, 2);
*cmds++ = adreno_getreg(adreno_dev,
ADRENO_REG_RBBM_SECVID_TRUST_CONTROL);
*cmds++ = set;
*cmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
*cmds++ = 1;
/* Stall PFP until all above commands are complete */
cmds += cp_wait_for_me(adreno_dev, cmds);
} else {
/*
* A5xx has a separate opcode specifically to put the GPU
* in and out of secure mode.
*/
*cmds++ = cp_packet(adreno_dev, CP_SET_SECURE_MODE, 1);
*cmds++ = set;
}
return cmds - start;
}
static int
adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
unsigned int flags, unsigned int *cmds,
unsigned int sizedwords, uint32_t timestamp,
struct adreno_submit_time *time)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
struct kgsl_device *device = rb->device;
unsigned int *ringcmds, *start;
unsigned int total_sizedwords = sizedwords;
unsigned int i;
unsigned int context_id = 0;
uint64_t gpuaddr = rb->device->memstore.gpuaddr;
bool profile_ready;
struct adreno_context *drawctxt = rb->drawctxt_active;
struct kgsl_context *context = NULL;
bool secured_ctxt = false;
uint64_t cond_addr;
if (drawctxt != NULL && kgsl_context_detached(&drawctxt->base) &&
!(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE))
return -ENOENT;
rb->timestamp++;
/* If this is a internal IB, use the global timestamp for it */
if (!drawctxt || (flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE))
timestamp = rb->timestamp;
else {
context_id = drawctxt->base.id;
context = &drawctxt->base;
}
/*
* Note that we cannot safely take drawctxt->mutex here without
* potential mutex inversion with device->mutex which is held
* here. As a result, any other code that accesses this variable
* must also use device->mutex.
*/
if (drawctxt) {
drawctxt->internal_timestamp = rb->timestamp;
if (drawctxt->base.flags & KGSL_CONTEXT_SECURE)
secured_ctxt = true;
}
/*
* If in stream ib profiling is enabled and there are counters
* assigned, then space needs to be reserved for profiling. This
* space in the ringbuffer is always consumed (might be filled with
* NOPs in error case. profile_ready needs to be consistent through
* the _addcmds call since it is allocating additional ringbuffer
* command space.
*/
profile_ready = drawctxt &&
adreno_profile_assignments_ready(&adreno_dev->profile) &&
!(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE);
/* reserve space to temporarily turn off protected mode
* error checking if needed
*/
total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
/* 2 dwords to store the start of command sequence */
total_sizedwords += 2;
/* internal ib command identifier for the ringbuffer */
total_sizedwords += (flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE) ? 2 : 0;
total_sizedwords += (secured_ctxt) ? 26 : 0;
/* context rollover */
if (adreno_is_a3xx(adreno_dev))
total_sizedwords += 3;
/* For HLSQ updates below */
if (adreno_is_a4xx(adreno_dev) || adreno_is_a3xx(adreno_dev))
total_sizedwords += 4;
if (gpudev->preemption_pre_ibsubmit &&
adreno_is_preemption_enabled(adreno_dev))
total_sizedwords += 20;
if (gpudev->preemption_post_ibsubmit &&
adreno_is_preemption_enabled(adreno_dev))
total_sizedwords += 13;
/*
* a5xx uses 64 bit memory address. pm4 commands that involve read/write
* from memory take 4 bytes more than a4xx because of 64 bit addressing.
* This function is shared between gpucores, so reserve the max size
* required in ringbuffer and adjust the write pointer depending on
* gpucore at the end of this function.
*/
total_sizedwords += 4; /* sop timestamp */
total_sizedwords += 5; /* eop timestamp */
if (drawctxt && !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE)) {
total_sizedwords += 4; /* global timestamp without cache
* flush for non-zero context */
}
if (flags & KGSL_CMD_FLAGS_WFI)
total_sizedwords += 2; /* WFI */
if (profile_ready)
total_sizedwords += 8; /* space for pre_ib and post_ib */
/* Add space for the power on shader fixup if we need it */
if (flags & KGSL_CMD_FLAGS_PWRON_FIXUP)
total_sizedwords += 9;
if (sizedwords >= KGSL_RB_DWORDS || total_sizedwords >= KGSL_RB_DWORDS || total_sizedwords > (sizedwords + 200)) {
unsigned int passed_size = sizedwords;
unsigned int updated_size = total_sizedwords ;
printk("adreno_ringbuffer_addcmds: sizedwords %d, total_sizedwords %d\n", passed_size, updated_size);
BUG();
}
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
if (IS_ERR(ringcmds))
return PTR_ERR(ringcmds);
start = ringcmds;
*ringcmds++ = cp_packet(adreno_dev, CP_NOP, 1);
*ringcmds++ = KGSL_CMD_IDENTIFIER;
if (adreno_is_preemption_enabled(adreno_dev) &&
gpudev->preemption_pre_ibsubmit) {
cond_addr = device->memstore.gpuaddr +
KGSL_MEMSTORE_OFFSET(context_id,
preempted);
ringcmds += gpudev->preemption_pre_ibsubmit(
adreno_dev, rb, ringcmds, context,
cond_addr, NULL);
}
if (flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE) {
*ringcmds++ = cp_packet(adreno_dev, CP_NOP, 1);
*ringcmds++ = KGSL_CMD_INTERNAL_IDENTIFIER;
}
if (flags & KGSL_CMD_FLAGS_PWRON_FIXUP) {
/* Disable protected mode for the fixup */
*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
*ringcmds++ = 0;
*ringcmds++ = cp_packet(adreno_dev, CP_NOP, 1);
*ringcmds++ = KGSL_PWRON_FIXUP_IDENTIFIER;
*ringcmds++ = cp_mem_packet(adreno_dev,
CP_INDIRECT_BUFFER_PFE, 2, 1);
ringcmds += cp_gpuaddr(adreno_dev, ringcmds,
adreno_dev->pwron_fixup.gpuaddr);
*ringcmds++ = adreno_dev->pwron_fixup_dwords;
/* Re-enable protected mode */
*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
*ringcmds++ = 1;
}
/* Add any IB required for profiling if it is enabled */
if (profile_ready)
adreno_profile_preib_processing(adreno_dev, drawctxt,
&flags, &ringcmds);
/* start-of-pipeline timestamp */
*ringcmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1);
if (drawctxt && !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE))
ringcmds += cp_gpuaddr(adreno_dev, ringcmds,
gpuaddr + KGSL_MEMSTORE_OFFSET(context_id,
soptimestamp));
else
ringcmds += cp_gpuaddr(adreno_dev, ringcmds,
gpuaddr + KGSL_MEMSTORE_RB_OFFSET(rb, soptimestamp));
*ringcmds++ = timestamp;
if (secured_ctxt)
ringcmds += cp_secure_mode(adreno_dev, ringcmds, 1);
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* disable protected mode error checking */
*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
*ringcmds++ = 0;
}
for (i = 0; i < sizedwords; i++)
*ringcmds++ = cmds[i];
if (flags & KGSL_CMD_FLAGS_PMODE) {
/* re-enable protected mode error checking */
*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
*ringcmds++ = 1;
}
/*
* Flush HLSQ lazy updates to make sure there are no
* resources pending for indirect loads after the timestamp
*/
if (adreno_is_a4xx(adreno_dev) || adreno_is_a3xx(adreno_dev)) {
*ringcmds++ = cp_packet(adreno_dev, CP_EVENT_WRITE, 1);
*ringcmds++ = 0x07; /* HLSQ_FLUSH */
ringcmds += cp_wait_for_idle(adreno_dev, ringcmds);
}
/* Add any postIB required for profiling if it is enabled and has
assigned counters */
if (profile_ready)
adreno_profile_postib_processing(adreno_dev, &flags, &ringcmds);
/*
* end-of-pipeline timestamp. If per context timestamps is not
* enabled, then drawctxt will be NULL or internal command flag will be
* set and hence the rb timestamp will be used in else statement below.
*/
*ringcmds++ = cp_mem_packet(adreno_dev, CP_EVENT_WRITE, 3, 1);
if (drawctxt || (flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE))
*ringcmds++ = CACHE_FLUSH_TS | (1 << 31);
else
*ringcmds++ = CACHE_FLUSH_TS;
if (drawctxt && !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE)) {
ringcmds += cp_gpuaddr(adreno_dev, ringcmds, gpuaddr +
KGSL_MEMSTORE_OFFSET(context_id, eoptimestamp));
*ringcmds++ = timestamp;
*ringcmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1);
ringcmds += cp_gpuaddr(adreno_dev, ringcmds, gpuaddr +
KGSL_MEMSTORE_RB_OFFSET(rb, eoptimestamp));
*ringcmds++ = rb->timestamp;
} else {
ringcmds += cp_gpuaddr(adreno_dev, ringcmds, gpuaddr +
KGSL_MEMSTORE_RB_OFFSET(rb, eoptimestamp));
*ringcmds++ = timestamp;
}
if (adreno_is_a3xx(adreno_dev)) {
/* Dummy set-constant to trigger context rollover */
*ringcmds++ = cp_packet(adreno_dev, CP_SET_CONSTANT, 2);
*ringcmds++ =
(0x4<<16) | (A3XX_HLSQ_CL_KERNEL_GROUP_X_REG - 0x2000);
*ringcmds++ = 0;
}
if (flags & KGSL_CMD_FLAGS_WFI) {
ringcmds += cp_wait_for_idle(adreno_dev, ringcmds);
}
if (secured_ctxt)
ringcmds += cp_secure_mode(adreno_dev, ringcmds, 0);
if (gpudev->preemption_post_ibsubmit &&
adreno_is_preemption_enabled(adreno_dev))
ringcmds += gpudev->preemption_post_ibsubmit(adreno_dev,
rb, ringcmds, &drawctxt->base);
/*
* If we have more ringbuffer commands than space reserved
* in ringbuffer BUG() to fix this because it will lead to
* weird errors.
*/
if ((ringcmds - start) > total_sizedwords)
BUG();
/*
* Allocate total_sizedwords space in RB, this is the max space
* required. If we have commands less than the space reserved in RB
* adjust the wptr accordingly.
*/
rb->wptr = rb->wptr - (total_sizedwords - (ringcmds - start));
adreno_ringbuffer_submit(rb, time);
return 0;
}
int
adreno_ringbuffer_issuecmds(struct adreno_ringbuffer *rb,
unsigned int flags,
unsigned int *cmds,
int sizedwords)
{
flags |= KGSL_CMD_FLAGS_INTERNAL_ISSUE;
return adreno_ringbuffer_addcmds(rb, flags, cmds,
sizedwords, 0, NULL);
}
/**
* _ringbuffer_verify_ib() - Check if an IB's size is within a permitted limit
* @device: The kgsl device pointer
* @ibdesc: Pointer to the IB descriptor
*/
static inline bool _ringbuffer_verify_ib(struct kgsl_device_private *dev_priv,
struct kgsl_context *context, struct kgsl_memobj_node *ib)
{
struct kgsl_device *device = dev_priv->device;
struct kgsl_process_private *private = dev_priv->process_priv;
/* The maximum allowable size for an IB in the CP is 0xFFFFF dwords */
if (ib->size == 0 || ((ib->size >> 2) > 0xFFFFF)) {
pr_context(device, context, "ctxt %d invalid ib size %lld\n",
context->id, ib->size);
return false;
}
/* Make sure that the address is mapped */
if (!kgsl_mmu_gpuaddr_in_range(private->pagetable, ib->gpuaddr)) {
pr_context(device, context, "ctxt %d invalid ib gpuaddr %llX\n",
context->id, ib->gpuaddr);
return false;
}
return true;
}
int
adreno_ringbuffer_issueibcmds(struct kgsl_device_private *dev_priv,
struct kgsl_context *context,
struct kgsl_cmdbatch *cmdbatch,
uint32_t *timestamp)
{
struct kgsl_device *device = dev_priv->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
struct kgsl_memobj_node *ib;
int ret;
if (kgsl_context_invalid(context))
return -EDEADLK;
/* Verify the IBs before they get queued */
list_for_each_entry(ib, &cmdbatch->cmdlist, node)
if (_ringbuffer_verify_ib(dev_priv, context, ib) == false)
return -EINVAL;
/* wait for the suspend gate */
wait_for_completion(&device->cmdbatch_gate);
/*
* Clear the wake on touch bit to indicate an IB has been
* submitted since the last time we set it. But only clear
* it when we have rendering commands.
*/
if (!(cmdbatch->flags & KGSL_CMDBATCH_MARKER)
&& !(cmdbatch->flags & KGSL_CMDBATCH_SYNC))
device->flags &= ~KGSL_FLAG_WAKE_ON_TOUCH;
/* Queue the command in the ringbuffer */
ret = adreno_dispatcher_queue_cmd(adreno_dev, drawctxt, cmdbatch,
timestamp);
/*
* Return -EPROTO if the device has faulted since the last time we
* checked - userspace uses this to perform post-fault activities
*/
if (!ret && test_and_clear_bit(ADRENO_CONTEXT_FAULT, &context->priv))
ret = -EPROTO;
return ret;
}
static void adreno_ringbuffer_set_constraint(struct kgsl_device *device,
struct kgsl_cmdbatch *cmdbatch)
{
struct kgsl_context *context = cmdbatch->context;
/*
* Check if the context has a constraint and constraint flags are
* set.
*/
if (context->pwr_constraint.type &&
((context->flags & KGSL_CONTEXT_PWR_CONSTRAINT) ||
(cmdbatch->flags & KGSL_CONTEXT_PWR_CONSTRAINT)))
kgsl_pwrctrl_set_constraint(device, &context->pwr_constraint,
context->id);
}
static inline int _get_alwayson_counter(struct adreno_device *adreno_dev,
unsigned int *cmds, uint64_t gpuaddr)
{
unsigned int *p = cmds;
*p++ = cp_mem_packet(adreno_dev, CP_REG_TO_MEM, 2, 1);
/*
* For a4x and some a5x the alwayson_hi read through CPU
* will be masked. Only do 32 bit CP reads for keeping the
* numbers consistent
*/
if (ADRENO_GPUREV(adreno_dev) >= 400 &&
ADRENO_GPUREV(adreno_dev) <= ADRENO_REV_A530)
*p++ = adreno_getreg(adreno_dev,
ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO);
else
*p++ = adreno_getreg(adreno_dev,
ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO) |
(1 << 30) | (2 << 18);
p += cp_gpuaddr(adreno_dev, p, gpuaddr);
return (unsigned int)(p - cmds);
}
/* adreno_rindbuffer_submitcmd - submit userspace IBs to the GPU */
int adreno_ringbuffer_submitcmd(struct adreno_device *adreno_dev,
struct kgsl_cmdbatch *cmdbatch, struct adreno_submit_time *time)
{
struct kgsl_device *device = &adreno_dev->dev;
struct kgsl_memobj_node *ib;
unsigned int numibs = 0;
unsigned int *link;
unsigned int *cmds;
struct kgsl_context *context;
struct adreno_context *drawctxt;
bool use_preamble = true;
bool cmdbatch_user_profiling = false;
bool cmdbatch_kernel_profiling = false;
int flags = KGSL_CMD_FLAGS_NONE;
int ret;
struct adreno_ringbuffer *rb;
struct kgsl_cmdbatch_profiling_buffer *profile_buffer = NULL;
unsigned int dwords = 0;
struct adreno_submit_time local;
struct kgsl_mem_entry *entry = cmdbatch->profiling_buf_entry;
if (entry)
profile_buffer = kgsl_gpuaddr_to_vaddr(&entry->memdesc,
cmdbatch->profiling_buffer_gpuaddr);
context = cmdbatch->context;
drawctxt = ADRENO_CONTEXT(context);
/* Get the total IBs in the list */
list_for_each_entry(ib, &cmdbatch->cmdlist, node)
numibs++;
rb = drawctxt->rb;
/* process any profiling results that are available into the log_buf */
adreno_profile_process_results(adreno_dev);
/*
* If SKIP CMD flag is set for current context
* a) set SKIPCMD as fault_recovery for current commandbatch
* b) store context's commandbatch fault_policy in current
* commandbatch fault_policy and clear context's commandbatch
* fault_policy
* c) force preamble for commandbatch
*/
if (test_bit(ADRENO_CONTEXT_SKIP_CMD, &drawctxt->base.priv) &&
(!test_bit(CMDBATCH_FLAG_SKIP, &cmdbatch->priv))) {
set_bit(KGSL_FT_SKIPCMD, &cmdbatch->fault_recovery);
cmdbatch->fault_policy = drawctxt->fault_policy;
set_bit(CMDBATCH_FLAG_FORCE_PREAMBLE, &cmdbatch->priv);
/* if context is detached print fault recovery */
adreno_fault_skipcmd_detached(device, drawctxt, cmdbatch);
/* clear the drawctxt flags */
clear_bit(ADRENO_CONTEXT_SKIP_CMD, &drawctxt->base.priv);
drawctxt->fault_policy = 0;
}
/*When preamble is enabled, the preamble buffer with state restoration
commands are stored in the first node of the IB chain. We can skip that
if a context switch hasn't occured */
if ((drawctxt->base.flags & KGSL_CONTEXT_PREAMBLE) &&
!test_bit(CMDBATCH_FLAG_FORCE_PREAMBLE, &cmdbatch->priv) &&
(rb->drawctxt_active == drawctxt))
use_preamble = false;
/*
* In skip mode don't issue the draw IBs but keep all the other
* accoutrements of a submision (including the interrupt) to keep
* the accounting sane. Set start_index and numibs to 0 to just
* generate the start and end markers and skip everything else
*/
if (test_bit(CMDBATCH_FLAG_SKIP, &cmdbatch->priv)) {
use_preamble = false;
numibs = 0;
}
/*
* a5xx uses 64 bit memory address. pm4 commands that involve read/write
* from memory take 4 bytes more than a4xx because of 64 bit addressing.
* This function is shared between gpucores, so reserve the max size
* required and adjust the number of commands before calling addcmds.
* Each submission needs 7 dwords max for wrappers and other red tape.
*/
dwords = 7;
/* Each IB takes up 30 dwords in worst case */
dwords += (numibs * 30);
if (cmdbatch->flags & KGSL_CMDBATCH_PROFILING &&
!adreno_is_a3xx(adreno_dev) && profile_buffer) {
cmdbatch_user_profiling = true;
dwords += 6;
/*
* REG_TO_MEM packet on A5xx needs another ordinal.
* Add 2 more dwords since we do profiling before and after.
*/
if (adreno_is_a5xx(adreno_dev))
dwords += 2;
/*
* we want to use an adreno_submit_time struct to get the
* precise moment when the command is submitted to the
* ringbuffer. If an upstream caller already passed down a
* pointer piggyback on that otherwise use a local struct
*/
if (time == NULL)
time = &local;
}
if (test_bit(CMDBATCH_FLAG_PROFILE, &cmdbatch->priv)) {
cmdbatch_kernel_profiling = true;
dwords += 6;
if (adreno_is_a5xx(adreno_dev))
dwords += 2;
}
link = kzalloc(sizeof(unsigned int) * dwords, GFP_KERNEL);
if (!link) {
ret = -ENOMEM;
goto done;
}
cmds = link;
*cmds++ = cp_packet(adreno_dev, CP_NOP, 1);
*cmds++ = KGSL_START_OF_IB_IDENTIFIER;
if (cmdbatch_kernel_profiling) {
cmds += _get_alwayson_counter(adreno_dev, cmds,
adreno_dev->cmdbatch_profile_buffer.gpuaddr +
ADRENO_CMDBATCH_PROFILE_OFFSET(cmdbatch->profile_index,
started));
}
/*
* Add cmds to read the GPU ticks at the start of the cmdbatch and
* write it into the appropriate cmdbatch profiling buffer offset
*/
if (cmdbatch_user_profiling) {
cmds += _get_alwayson_counter(adreno_dev, cmds,
cmdbatch->profiling_buffer_gpuaddr +
offsetof(struct kgsl_cmdbatch_profiling_buffer,
gpu_ticks_submitted));
}
if (numibs) {
list_for_each_entry(ib, &cmdbatch->cmdlist, node) {
/*
* Skip 0 sized IBs - these are presumed to have been
* removed from consideration by the FT policy
*/
if (ib->priv & MEMOBJ_SKIP ||
(ib->priv & MEMOBJ_PREAMBLE &&
use_preamble == false))
*cmds++ = cp_mem_packet(adreno_dev, CP_NOP,
3, 1);
*cmds++ = cp_mem_packet(adreno_dev,
CP_INDIRECT_BUFFER_PFE, 2, 1);
cmds += cp_gpuaddr(adreno_dev, cmds, ib->gpuaddr);
*cmds++ = (unsigned int) ib->size >> 2;
/* preamble is required on only for first command */
use_preamble = false;
}
}
if (cmdbatch_kernel_profiling) {
cmds += _get_alwayson_counter(adreno_dev, cmds,
adreno_dev->cmdbatch_profile_buffer.gpuaddr +
ADRENO_CMDBATCH_PROFILE_OFFSET(cmdbatch->profile_index,
retired));
}
/*
* Add cmds to read the GPU ticks at the end of the cmdbatch and
* write it into the appropriate cmdbatch profiling buffer offset
*/
if (cmdbatch_user_profiling) {
cmds += _get_alwayson_counter(adreno_dev, cmds,
cmdbatch->profiling_buffer_gpuaddr +
offsetof(struct kgsl_cmdbatch_profiling_buffer,
gpu_ticks_retired));
}
*cmds++ = cp_packet(adreno_dev, CP_NOP, 1);
*cmds++ = KGSL_END_OF_IB_IDENTIFIER;
ret = adreno_drawctxt_switch(adreno_dev, rb, drawctxt, cmdbatch->flags);
/*
* In the unlikely event of an error in the drawctxt switch,
* treat it like a hang
*/
if (ret)
goto done;
if (test_bit(CMDBATCH_FLAG_WFI, &cmdbatch->priv))
flags = KGSL_CMD_FLAGS_WFI;
/*
* For some targets, we need to execute a dummy shader operation after a
* power collapse
*/
if (test_and_clear_bit(ADRENO_DEVICE_PWRON, &adreno_dev->priv) &&
test_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv))
flags |= KGSL_CMD_FLAGS_PWRON_FIXUP;
/* Set the constraints before adding to ringbuffer */
adreno_ringbuffer_set_constraint(device, cmdbatch);
/* CFF stuff executed only if CFF is enabled */
kgsl_cffdump_capture_ib_desc(device, context, cmdbatch);
ret = adreno_ringbuffer_addcmds(rb, flags,
&link[0], (cmds - link),
cmdbatch->timestamp, time);
if (!ret) {
cmdbatch->global_ts = drawctxt->internal_timestamp;
/* Put the timevalues in the profiling buffer */
if (cmdbatch_user_profiling) {
profile_buffer->wall_clock_s = time->utime.tv_sec;
profile_buffer->wall_clock_ns = time->utime.tv_nsec;
profile_buffer->gpu_ticks_queued = time->ticks;
}
}
kgsl_cffdump_regpoll(device,
adreno_getreg(adreno_dev, ADRENO_REG_RBBM_STATUS) << 2,
0x00000000, 0x80000000);
done:
/* Corresponding unmap to the memdesc map of profile_buffer */
if (entry)
kgsl_memdesc_unmap(&entry->memdesc);
trace_kgsl_issueibcmds(device, context->id, cmdbatch,
numibs, cmdbatch->timestamp,
cmdbatch->flags, ret, drawctxt->type);
kfree(link);
return ret;
}
/**
* adreno_ringbuffer_mmu_clk_disable_event() - Callback function that
* disables the MMU clocks.
* @device: Device pointer
* @context: The ringbuffer context pointer
* @data: Pointer containing the adreno_mmu_disable_clk_param structure
* @type: The event call type (RETIRED or CANCELLED)
*/
static void adreno_ringbuffer_mmu_clk_disable_event(struct kgsl_device *device,
struct kgsl_event_group *group, void *data, int type)
{
kgsl_mmu_disable_clk(&device->mmu);
}
/*
* adreno_ringbuffer_mmu_disable_clk_on_ts() - Sets up event to disable MMU
* clocks
* @device - The kgsl device pointer
* @rb: The ringbuffer in whose event list the event is added
* @timestamp: The timestamp on which the event should trigger
*
* Creates an event to disable the MMU clocks on timestamp and if event
* already exists then updates the timestamp of disabling the MMU clocks
* with the passed in ts if it is greater than the current value at which
* the clocks will be disabled
* Return - void
*/
void
adreno_ringbuffer_mmu_disable_clk_on_ts(struct kgsl_device *device,
struct adreno_ringbuffer *rb, unsigned int timestamp)
{
if (kgsl_add_event(device, &(rb->events), timestamp,
adreno_ringbuffer_mmu_clk_disable_event, NULL)) {
KGSL_DRV_ERR(device,
"Failed to add IOMMU disable clk event\n");
}
}
/**
* adreno_ringbuffer_wait_callback() - Callback function for event registered
* on a ringbuffer timestamp
* @device: Device for which the the callback is valid
* @context: The context of the event
* @priv: The private parameter of the event
* @result: Result of the event trigger
*/
static void adreno_ringbuffer_wait_callback(struct kgsl_device *device,
struct kgsl_event_group *group,
void *priv, int result)
{
struct adreno_ringbuffer *rb = group->priv;
wake_up_all(&rb->ts_expire_waitq);
}
/**
* adreno_ringbuffer_waittimestamp() - Wait for a RB timestamp
* @rb: The ringbuffer to wait on
* @timestamp: The timestamp to wait for
* @msecs: The wait timeout period
*/
int adreno_ringbuffer_waittimestamp(struct adreno_ringbuffer *rb,
unsigned int timestamp,
unsigned int msecs)
{
struct kgsl_device *device = rb->device;
int ret;
unsigned long wait_time;
/* force a timeout from caller for the wait */
BUG_ON(0 == msecs);
ret = kgsl_add_event(device, &rb->events, timestamp,
adreno_ringbuffer_wait_callback, NULL);
if (ret)
return ret;
mutex_unlock(&device->mutex);
wait_time = msecs_to_jiffies(msecs);
if (0 == wait_event_timeout(rb->ts_expire_waitq,
!kgsl_event_pending(device, &rb->events, timestamp,
adreno_ringbuffer_wait_callback, NULL),
wait_time))
ret = -ETIMEDOUT;
mutex_lock(&device->mutex);
/*
* after wake up make sure that expected timestamp has retired
* because the wakeup could have happened due to a cancel event
*/
if (!ret && !adreno_ringbuffer_check_timestamp(rb,
timestamp, KGSL_TIMESTAMP_RETIRED)) {
ret = -EAGAIN;
}
return ret;
}
/**
* adreno_ringbuffer_submit_preempt_token() - Submit a preempt token
* @rb: Ringbuffer in which the token is submitted
* @incoming_rb: The RB to which the GPU switches when this preemption
* token is executed.
*
* Called to make sure that an outstanding preemption request is
* granted.
*/
int adreno_ringbuffer_submit_preempt_token(struct adreno_ringbuffer *rb,
struct adreno_ringbuffer *incoming_rb)
{
unsigned int *ringcmds, *start;
struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
struct kgsl_device *device = &(adreno_dev->dev);
struct kgsl_iommu *iommu = device->mmu.priv;
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
int ptname;
struct kgsl_pagetable *pt;
int pt_switch_sizedwords = 0, total_sizedwords = 20;
unsigned link[ADRENO_RB_PREEMPT_TOKEN_DWORDS];
uint i;
uint64_t ttbr0;
if (incoming_rb->preempted_midway) {
if (adreno_is_a5xx(adreno_dev)) {
kgsl_sharedmem_readq(&rb->pagetable_desc, &ttbr0,
offsetof(struct adreno_ringbuffer_pagetable_info
, ttbr0));
kgsl_sharedmem_writeq(rb->device, &iommu->smmu_info,
offsetof(struct a5xx_cp_smmu_info, ttbr0),
ttbr0);
} else {
kgsl_sharedmem_readl(&incoming_rb->pagetable_desc,
&ptname, offsetof(
struct adreno_ringbuffer_pagetable_info,
current_rb_ptname));
pt = kgsl_mmu_get_pt_from_ptname(&(rb->device->mmu),
ptname);
/*
* always expect a valid pt, else pt refcounting is
* messed up or current pt tracking has a bug which
* could lead to eventual disaster
*/
BUG_ON(!pt);
/* set the ringbuffer for incoming RB */
pt_switch_sizedwords =
adreno_iommu_set_pt_generate_cmds(incoming_rb,
&link[0], pt);
total_sizedwords += pt_switch_sizedwords;
}
}
/*
* Allocate total_sizedwords space in RB, this is the max space
* required.
*/
ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
if (IS_ERR(ringcmds))
return PTR_ERR(ringcmds);
start = ringcmds;
*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
*ringcmds++ = 0;
if (incoming_rb->preempted_midway) {
for (i = 0; i < pt_switch_sizedwords; i++)
*ringcmds++ = link[i];
}
*ringcmds++ = cp_register(adreno_dev, adreno_getreg(adreno_dev,
ADRENO_REG_CP_PREEMPT_DISABLE), 1);
*ringcmds++ = 0;
*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
*ringcmds++ = 1;
ringcmds += gpudev->preemption_token(adreno_dev, rb, ringcmds,
rb->device->memstore.gpuaddr +
KGSL_MEMSTORE_RB_OFFSET(rb, preempted));
if ((uint)(ringcmds - start) > total_sizedwords) {
KGSL_DRV_ERR(device, "Insufficient rb size allocated\n");
BUG();
}
/*
* If we have commands less than the space reserved in RB
* adjust the wptr accordingly
*/
rb->wptr = rb->wptr - (total_sizedwords - (uint)(ringcmds - start));
/* submit just the preempt token */
mb();
kgsl_pwrscale_busy(rb->device);
adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, rb->wptr);
return 0;
}
Reference in a new issue View git blame Copy permalink