Merge 388cd19afe
on remote branch
Change-Id: Ia2736fe53d507fc2464ee67c7a82ea456c07a04a
This commit is contained in:
commit
2403a5e379
12 changed files with 131 additions and 42 deletions
|
@ -1,7 +1,7 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2008-2021, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
* Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
#ifndef __ADRENO_H
|
||||
#define __ADRENO_H
|
||||
|
@ -16,9 +16,6 @@
|
|||
#define DEVICE_3D_NAME "kgsl-3d"
|
||||
#define DEVICE_3D0_NAME "kgsl-3d0"
|
||||
|
||||
/* Index to preemption scratch buffer to store KMD postamble */
|
||||
#define KMD_POSTAMBLE_IDX 100
|
||||
|
||||
/* ADRENO_DEVICE - Given a kgsl_device return the adreno device struct */
|
||||
#define ADRENO_DEVICE(device) \
|
||||
container_of(device, struct adreno_device, dev)
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
* Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include "adreno.h"
|
||||
|
@ -558,8 +558,8 @@ unsigned int a6xx_preemption_pre_ibsubmit(
|
|||
* preemption
|
||||
*/
|
||||
if (!adreno_dev->perfcounter) {
|
||||
u64 kmd_postamble_addr =
|
||||
PREEMPT_SCRATCH_ADDR(adreno_dev, KMD_POSTAMBLE_IDX);
|
||||
u64 kmd_postamble_addr = SCRATCH_POSTAMBLE_ADDR
|
||||
(KGSL_DEVICE(adreno_dev));
|
||||
|
||||
*cmds++ = cp_type7_packet(CP_SET_AMBLE, 3);
|
||||
*cmds++ = lower_32_bits(kmd_postamble_addr);
|
||||
|
@ -763,6 +763,8 @@ void a6xx_preemption_close(struct adreno_device *adreno_dev)
|
|||
|
||||
int a6xx_preemption_init(struct adreno_device *adreno_dev)
|
||||
{
|
||||
u32 flags = ADRENO_FEATURE(adreno_dev, ADRENO_APRIV) ?
|
||||
KGSL_MEMDESC_PRIVILEGED : 0;
|
||||
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
|
||||
struct adreno_preemption *preempt = &adreno_dev->preempt;
|
||||
struct adreno_ringbuffer *rb;
|
||||
|
@ -777,8 +779,8 @@ int a6xx_preemption_init(struct adreno_device *adreno_dev)
|
|||
|
||||
timer_setup(&preempt->timer, _a6xx_preemption_timer, 0);
|
||||
|
||||
ret = kgsl_allocate_global(device, &preempt->scratch, PAGE_SIZE, 0, 0,
|
||||
"preemption_scratch");
|
||||
ret = kgsl_allocate_global(device, &preempt->scratch, PAGE_SIZE, 0,
|
||||
flags, "preemption_scratch");
|
||||
|
||||
/* Allocate mem for storing preemption switch record */
|
||||
FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
|
||||
|
@ -788,14 +790,15 @@ int a6xx_preemption_init(struct adreno_device *adreno_dev)
|
|||
}
|
||||
|
||||
/*
|
||||
* First 8 dwords of the preemption scratch buffer is used to store the
|
||||
* address for CP to save/restore VPC data. Reserve 11 dwords in the
|
||||
* preemption scratch buffer from index KMD_POSTAMBLE_IDX for KMD
|
||||
* postamble pm4 packets
|
||||
* First 28 dwords of the device scratch buffer are used to store
|
||||
* shadow rb data. Reserve 11 dwords in the device scratch buffer
|
||||
* from SCRATCH_POSTAMBLE_OFFSET for KMD postamble pm4 packets.
|
||||
* This should be in *device->scratch* so that userspace cannot
|
||||
* access it.
|
||||
*/
|
||||
if (!adreno_dev->perfcounter) {
|
||||
u32 *postamble = preempt->scratch.hostptr +
|
||||
(KMD_POSTAMBLE_IDX * sizeof(u64));
|
||||
u32 *postamble = device->scratch.hostptr +
|
||||
SCRATCH_POSTAMBLE_OFFSET;
|
||||
u32 count = 0;
|
||||
|
||||
postamble[count++] = cp_type7_packet(CP_REG_RMW, 3);
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2008-2021, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
#ifndef __KGSL_H
|
||||
#define __KGSL_H
|
||||
|
@ -69,6 +70,11 @@
|
|||
#define SCRATCH_RPTR_GPU_ADDR(dev, id) \
|
||||
((dev)->scratch.gpuaddr + SCRATCH_RPTR_OFFSET(id))
|
||||
|
||||
/* OFFSET to KMD postamble packets in scratch buffer */
|
||||
#define SCRATCH_POSTAMBLE_OFFSET (100 * sizeof(u64))
|
||||
#define SCRATCH_POSTAMBLE_ADDR(dev) \
|
||||
((dev)->scratch.gpuaddr + SCRATCH_POSTAMBLE_OFFSET)
|
||||
|
||||
/* Timestamp window used to detect rollovers (half of integer range) */
|
||||
#define KGSL_TIMESTAMP_WINDOW 0x80000000
|
||||
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/* Copyright (c) 2014-2015, 2017-2019, The Linux Foundation. All rights reserved. */
|
||||
/* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
|
@ -143,8 +144,11 @@ static int pmic_spmi_probe(struct spmi_device *sdev)
|
|||
|
||||
MODULE_DEVICE_TABLE(of, pmic_spmi_id_table);
|
||||
|
||||
static void pmic_spmi_remove(struct spmi_device *sdev) {}
|
||||
|
||||
static struct spmi_driver pmic_spmi_driver = {
|
||||
.probe = pmic_spmi_probe,
|
||||
.remove = pmic_spmi_remove,
|
||||
.driver = {
|
||||
.name = "pmic-spmi",
|
||||
.of_match_table = pmic_spmi_id_table,
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/cdev.h>
|
||||
|
@ -125,7 +126,6 @@ static int qiib_driver_data_init(void)
|
|||
*/
|
||||
static void qiib_driver_data_deinit(void)
|
||||
{
|
||||
qiib_cleanup();
|
||||
if (!qiib_info->log_ctx)
|
||||
ipc_log_context_destroy(qiib_info->log_ctx);
|
||||
kfree(qiib_info);
|
||||
|
@ -433,8 +433,10 @@ static void qiib_cleanup(void)
|
|||
}
|
||||
mutex_unlock(&qiib_info->list_lock);
|
||||
|
||||
if (!IS_ERR_OR_NULL(qiib_info->classp))
|
||||
if (!IS_ERR_OR_NULL(qiib_info->classp)) {
|
||||
class_destroy(qiib_info->classp);
|
||||
qiib_info->classp = NULL;
|
||||
}
|
||||
|
||||
unregister_chrdev_region(MAJOR(qiib_info->dev_num), qiib_info->nports);
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2016-2021, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
* Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/bitmap.h>
|
||||
|
@ -3601,6 +3601,7 @@ static int msm_geni_serial_remove(struct platform_device *pdev)
|
|||
|
||||
if (!uart_console(&port->uport)) {
|
||||
wakeup_source_unregister(port->geni_wake);
|
||||
port->geni_wake = NULL;
|
||||
flush_workqueue(port->qwork);
|
||||
destroy_workqueue(port->qwork);
|
||||
}
|
||||
|
|
|
@ -1498,6 +1498,12 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
|
|||
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
|
||||
static inline void vm_write_begin(struct vm_area_struct *vma)
|
||||
{
|
||||
/*
|
||||
* Isolated vma might be freed without exclusive mmap_lock but
|
||||
* speculative page fault handler still needs to know it was changed.
|
||||
*/
|
||||
if (!RB_EMPTY_NODE(&vma->vm_rb))
|
||||
WARN_ON_ONCE(!rwsem_is_locked(&(vma->vm_mm)->mmap_sem));
|
||||
/*
|
||||
* The reads never spins and preemption
|
||||
* disablement is not required.
|
||||
|
|
50
mm/filemap.c
50
mm/filemap.c
|
@ -2633,7 +2633,9 @@ static struct file *do_async_mmap_readahead(struct vm_fault *vmf,
|
|||
* it in the page cache, and handles the special cases reasonably without
|
||||
* having a lot of duplicated code.
|
||||
*
|
||||
* vma->vm_mm->mmap_sem must be held on entry (except FAULT_FLAG_SPECULATIVE).
|
||||
* If FAULT_FLAG_SPECULATIVE is set, this function runs with elevated vma
|
||||
* refcount and with mmap lock not held.
|
||||
* Otherwise, vma->vm_mm->mmap_sem must be held on entry.
|
||||
*
|
||||
* If our return value has VM_FAULT_RETRY set, it's because
|
||||
* lock_page_or_retry() returned 0.
|
||||
|
@ -2658,6 +2660,52 @@ vm_fault_t filemap_fault(struct vm_fault *vmf)
|
|||
struct page *page;
|
||||
vm_fault_t ret = 0;
|
||||
|
||||
if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
|
||||
page = find_get_page(mapping, offset);
|
||||
if (unlikely(!page))
|
||||
return VM_FAULT_RETRY;
|
||||
|
||||
if (unlikely(PageReadahead(page)))
|
||||
goto page_put;
|
||||
|
||||
if (!trylock_page(page))
|
||||
goto page_put;
|
||||
|
||||
if (unlikely(compound_head(page)->mapping != mapping))
|
||||
goto page_unlock;
|
||||
VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page);
|
||||
if (unlikely(!PageUptodate(page)))
|
||||
goto page_unlock;
|
||||
|
||||
max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
|
||||
if (unlikely(offset >= max_off))
|
||||
goto page_unlock;
|
||||
|
||||
/*
|
||||
* Update readahead mmap_miss statistic.
|
||||
*
|
||||
* Note that we are not sure if finish_fault() will
|
||||
* manage to complete the transaction. If it fails,
|
||||
* we'll come back to filemap_fault() non-speculative
|
||||
* case which will update mmap_miss a second time.
|
||||
* This is not ideal, we would prefer to guarantee the
|
||||
* update will happen exactly once.
|
||||
*/
|
||||
if (!(vmf->vma->vm_flags & VM_RAND_READ) && ra->ra_pages) {
|
||||
unsigned int mmap_miss = READ_ONCE(ra->mmap_miss);
|
||||
if (mmap_miss)
|
||||
WRITE_ONCE(ra->mmap_miss, --mmap_miss);
|
||||
}
|
||||
|
||||
vmf->page = page;
|
||||
return VM_FAULT_LOCKED;
|
||||
page_unlock:
|
||||
unlock_page(page);
|
||||
page_put:
|
||||
put_page(page);
|
||||
return VM_FAULT_RETRY;
|
||||
}
|
||||
|
||||
max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
|
||||
if (unlikely(offset >= max_off))
|
||||
return VM_FAULT_SIGBUS;
|
||||
|
|
|
@ -1294,10 +1294,12 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
|
|||
*/
|
||||
if (down_write_trylock(&mm->mmap_sem)) {
|
||||
if (!khugepaged_test_exit(mm)) {
|
||||
vm_write_begin(vma);
|
||||
spinlock_t *ptl = pmd_lock(mm, pmd);
|
||||
/* assume page table is clear */
|
||||
_pmd = pmdp_collapse_flush(vma, addr, pmd);
|
||||
spin_unlock(ptl);
|
||||
vm_write_end(vma);
|
||||
mm_dec_nr_ptes(mm);
|
||||
pte_free(mm, pmd_pgtable(_pmd));
|
||||
}
|
||||
|
|
38
mm/memory.c
38
mm/memory.c
|
@ -1553,7 +1553,6 @@ void unmap_page_range(struct mmu_gather *tlb,
|
|||
unsigned long next;
|
||||
|
||||
BUG_ON(addr >= end);
|
||||
vm_write_begin(vma);
|
||||
tlb_start_vma(tlb, vma);
|
||||
pgd = pgd_offset(vma->vm_mm, addr);
|
||||
do {
|
||||
|
@ -1563,7 +1562,6 @@ void unmap_page_range(struct mmu_gather *tlb,
|
|||
next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
|
||||
} while (pgd++, addr = next, addr != end);
|
||||
tlb_end_vma(tlb, vma);
|
||||
vm_write_end(vma);
|
||||
}
|
||||
|
||||
|
||||
|
@ -3207,6 +3205,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
|
|||
int exclusive = 0;
|
||||
vm_fault_t ret;
|
||||
|
||||
if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
|
||||
pte_unmap(vmf->pte);
|
||||
return VM_FAULT_RETRY;
|
||||
}
|
||||
|
||||
ret = pte_unmap_same(vmf);
|
||||
if (ret) {
|
||||
/*
|
||||
|
@ -3466,6 +3469,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
|
|||
if (vmf->vma_flags & VM_SHARED)
|
||||
return VM_FAULT_SIGBUS;
|
||||
|
||||
/* Do not check unstable pmd, if it's changed will retry later */
|
||||
if (vmf->flags & FAULT_FLAG_SPECULATIVE)
|
||||
goto skip_pmd_checks;
|
||||
|
||||
/*
|
||||
* Use pte_alloc() instead of pte_alloc_map(). We can't run
|
||||
* pte_offset_map() on pmds where a huge pmd might be created
|
||||
|
@ -3483,6 +3490,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
|
|||
if (unlikely(pmd_trans_unstable(vmf->pmd)))
|
||||
return 0;
|
||||
|
||||
skip_pmd_checks:
|
||||
/* Use the zero-page for reads */
|
||||
if (!(vmf->flags & FAULT_FLAG_WRITE) &&
|
||||
!mm_forbids_zeropage(vma->vm_mm)) {
|
||||
|
@ -3587,6 +3595,10 @@ static vm_fault_t __do_fault(struct vm_fault *vmf)
|
|||
struct vm_area_struct *vma = vmf->vma;
|
||||
vm_fault_t ret;
|
||||
|
||||
/* Do not check unstable pmd, if it's changed will retry later */
|
||||
if (vmf->flags & FAULT_FLAG_SPECULATIVE)
|
||||
goto skip_pmd_checks;
|
||||
|
||||
/*
|
||||
* Preallocate pte before we take page_lock because this might lead to
|
||||
* deadlocks for memcg reclaim which waits for pages under writeback:
|
||||
|
@ -3610,6 +3622,7 @@ static vm_fault_t __do_fault(struct vm_fault *vmf)
|
|||
smp_wmb(); /* See comment in __pte_alloc() */
|
||||
}
|
||||
|
||||
skip_pmd_checks:
|
||||
ret = vma->vm_ops->fault(vmf);
|
||||
if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
|
||||
VM_FAULT_DONE_COW)))
|
||||
|
@ -3988,7 +4001,8 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf)
|
|||
end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1,
|
||||
start_pgoff + nr_pages - 1);
|
||||
|
||||
if (pmd_none(*vmf->pmd)) {
|
||||
if (!(vmf->flags & FAULT_FLAG_SPECULATIVE) &&
|
||||
pmd_none(*vmf->pmd)) {
|
||||
vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm,
|
||||
vmf->address);
|
||||
if (!vmf->prealloc_pte)
|
||||
|
@ -4356,16 +4370,11 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
|
|||
pte_t entry;
|
||||
int ret = 0;
|
||||
|
||||
if (unlikely(pmd_none(*vmf->pmd))) {
|
||||
/*
|
||||
* In the case of the speculative page fault handler we abort
|
||||
* the speculative path immediately as the pmd is probably
|
||||
* in the way to be converted in a huge one. We will try
|
||||
* again holding the mmap_sem (which implies that the collapse
|
||||
* operation is done).
|
||||
*/
|
||||
/* Do not check unstable pmd, if it's changed will retry later */
|
||||
if (vmf->flags & FAULT_FLAG_SPECULATIVE)
|
||||
return VM_FAULT_RETRY;
|
||||
goto skip_pmd_checks;
|
||||
|
||||
if (unlikely(pmd_none(*vmf->pmd))) {
|
||||
/*
|
||||
* Leave __pte_alloc() until later: because vm_ops->fault may
|
||||
* want to allocate huge page, and if we expose page table
|
||||
|
@ -4373,8 +4382,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
|
|||
* concurrent faults and from rmap lookups.
|
||||
*/
|
||||
vmf->pte = NULL;
|
||||
} else if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) {
|
||||
/* See comment in pte_alloc_one_map() */
|
||||
} else {
|
||||
if (pmd_devmap_trans_unstable(vmf->pmd))
|
||||
return 0;
|
||||
/*
|
||||
|
@ -4404,6 +4412,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
|
|||
}
|
||||
}
|
||||
|
||||
skip_pmd_checks:
|
||||
if (!vmf->pte) {
|
||||
if (vma_is_anonymous(vmf->vma))
|
||||
return do_anonymous_page(vmf);
|
||||
|
@ -4651,7 +4660,6 @@ int __handle_speculative_fault(struct mm_struct *mm, unsigned long address,
|
|||
pol = __get_vma_policy(vmf.vma, address);
|
||||
if (!pol)
|
||||
pol = get_task_policy(current);
|
||||
if (!pol)
|
||||
if (pol && pol->mode == MPOL_INTERLEAVE) {
|
||||
trace_spf_vma_notsup(_RET_IP_, vmf.vma, address);
|
||||
return VM_FAULT_RETRY;
|
||||
|
|
|
@ -599,11 +599,9 @@ unsigned long change_prot_numa(struct vm_area_struct *vma,
|
|||
{
|
||||
int nr_updated;
|
||||
|
||||
vm_write_begin(vma);
|
||||
nr_updated = change_protection(vma, addr, end, PAGE_NONE, 0, 1);
|
||||
if (nr_updated)
|
||||
count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
|
||||
vm_write_end(vma);
|
||||
|
||||
return nr_updated;
|
||||
}
|
||||
|
|
18
mm/mmap.c
18
mm/mmap.c
|
@ -2306,8 +2306,22 @@ struct vm_area_struct *get_vma(struct mm_struct *mm, unsigned long addr)
|
|||
|
||||
read_lock(&mm->mm_rb_lock);
|
||||
vma = __find_vma(mm, addr);
|
||||
if (vma)
|
||||
atomic_inc(&vma->vm_ref_count);
|
||||
|
||||
/*
|
||||
* If there is a concurrent fast mremap, bail out since the entire
|
||||
* PMD/PUD subtree may have been remapped.
|
||||
*
|
||||
* This is usually safe for conventional mremap since it takes the
|
||||
* PTE locks as does SPF. However fast mremap only takes the lock
|
||||
* at the PMD/PUD level which is ok as it is done with the mmap
|
||||
* write lock held. But since SPF, as the term implies forgoes,
|
||||
* taking the mmap read lock and also cannot take PTL lock at the
|
||||
* larger PMD/PUD granualrity, since it would introduce huge
|
||||
* contention in the page fault path; fall back to regular fault
|
||||
* handling.
|
||||
*/
|
||||
if (vma && !atomic_inc_unless_negative(&vma->vm_ref_count))
|
||||
vma = NULL;
|
||||
read_unlock(&mm->mm_rb_lock);
|
||||
|
||||
return vma;
|
||||
|
|
Loading…
Reference in a new issue