90264576e2
* refs/heads/tmp-a483478: UPSTREAM: arm64: vdso: Build vDSO with -ffixed-x18 Revert "drm/dsi: Fix byte order of DCS set/get brightness" Reverting below patches from android-4.19-stable.125 Linux 4.19.125 rxrpc: Fix ack discard rxrpc: Trace discarded ACKs iio: adc: stm32-dfsdm: fix device used to request dma iio: adc: stm32-dfsdm: Use dma_request_chan() instead dma_request_slave_channel() iio: adc: stm32-adc: fix device used to request dma iio: adc: stm32-adc: Use dma_request_chan() instead dma_request_slave_channel() x86/unwind/orc: Fix unwind_get_return_address_ptr() for inactive tasks rxrpc: Fix a memory leak in rxkad_verify_response() rapidio: fix an error in get_user_pages_fast() error handling ipack: tpci200: fix error return code in tpci200_register() mei: release me_cl object reference misc: rtsx: Add short delay after exit from ASPM iio: dac: vf610: Fix an error handling path in 'vf610_dac_probe()' iio: sca3000: Remove an erroneous 'get_device()' staging: greybus: Fix uninitialized scalar variable staging: iio: ad2s1210: Fix SPI reading Revert "gfs2: Don't demote a glock until its revokes are written" brcmfmac: abort and release host after error tty: serial: qcom_geni_serial: Fix wrap around of TX buffer cxgb4/cxgb4vf: Fix mac_hlist initialization and free cxgb4: free mac_hlist properly net: bcmgenet: abort suspend on error net: bcmgenet: code movement Revert "net/ibmvnic: Fix EOI when running in XIVE mode" media: fdp1: Fix R-Car M3-N naming in debug message thunderbolt: Drop duplicated get_switch_at_route() staging: most: core: replace strcpy() by strscpy() libnvdimm/btt: Fix LBA masking during 'free list' population libnvdimm/btt: Remove unnecessary code in btt_freelist_init nfit: Add Hyper-V NVDIMM DSM command set to white list powerpc/64s: Disable STRICT_KERNEL_RWX powerpc: Remove STRICT_KERNEL_RWX incompatibility with RELOCATABLE drm/i915/gvt: Init DPLL/DDI vreg for virtual display instead of inheritance. dmaengine: owl: Use correct lock in owl_dma_get_pchan() dmaengine: tegra210-adma: Fix an error handling path in 'tegra_adma_probe()' apparmor: Fix aa_label refcnt leak in policy_update apparmor: fix potential label refcnt leak in aa_change_profile apparmor: Fix use-after-free in aa_audit_rule_init drm/etnaviv: fix perfmon domain interation ALSA: hda/realtek - Add more fixup entries for Clevo machines ALSA: hda/realtek - Fix silent output on Gigabyte X570 Aorus Xtreme ALSA: pcm: fix incorrect hw_base increase ALSA: iec1712: Initialize STDSP24 properly when using the model=staudio option padata: purge get_cpu and reorder_via_wq from padata_do_serial padata: initialize pd->cpu with effective cpumask padata: Replace delayed timer with immediate workqueue in padata_reorder ARM: futex: Address build warning platform/x86: asus-nb-wmi: Do not load on Asus T100TA and T200TA USB: core: Fix misleading driver bug report stmmac: fix pointer check after utilization in stmmac_interrupt ceph: fix double unlock in handle_cap_export() HID: quirks: Add HID_QUIRK_NO_INIT_REPORTS quirk for Dell K12A keyboard-dock gtp: set NLM_F_MULTI flag in gtp_genl_dump_pdp() x86/apic: Move TSC deadline timer debug printk HID: i2c-hid: reset Synaptics SYNA2393 on resume scsi: ibmvscsi: Fix WARN_ON during event pool release component: Silence bind error on -EPROBE_DEFER aquantia: Fix the media type of AQC100 ethernet controller in the driver vhost/vsock: fix packet delivery order to monitoring devices configfs: fix config_item refcnt leak in configfs_rmdir() scsi: qla2xxx: Delete all sessions before unregister local nvme port scsi: qla2xxx: Fix hang when issuing nvme disconnect-all in NPIV HID: alps: ALPS_1657 is too specific; use U1_UNICORN_LEGACY instead HID: alps: Add AUI1657 device ID HID: multitouch: add eGalaxTouch P80H84 support gcc-common.h: Update for GCC 10 ubi: Fix seq_file usage in detailed_erase_block_info debugfs file i2c: mux: demux-pinctrl: Fix an error handling path in 'i2c_demux_pinctrl_probe()' iommu/amd: Fix over-read of ACPI UID from IVRS table ubifs: remove broken lazytime support fix multiplication overflow in copy_fdtable() mtd: spinand: Propagate ECC information to the MTD structure ima: Fix return value of ima_write_policy() evm: Check also if *tfm is an error pointer in init_desc() ima: Set file->f_mode instead of file->f_flags in ima_calc_file_hash() riscv: set max_pfn to the PFN of the last page KVM: SVM: Fix potential memory leak in svm_cpu_init() i2c: dev: Fix the race between the release of i2c_dev and cdev ubsan: build ubsan.c more conservatively x86/uaccess, ubsan: Fix UBSAN vs. SMAP ANDROID: scsi: ufs: Handle clocks when lrbp fails ANDROID: fscrypt: handle direct I/O with IV_INO_LBLK_32 BACKPORT: FROMLIST: fscrypt: add support for IV_INO_LBLK_32 policies ANDROID: Update the ABI xml and qcom whitelist ANDROID: Fix build.config.gki-debug Linux 4.19.124 Makefile: disallow data races on gcc-10 as well KVM: x86: Fix off-by-one error in kvm_vcpu_ioctl_x86_setup_mce ARM: dts: r8a7740: Add missing extal2 to CPG node arm64: dts: renesas: r8a77980: Fix IPMMU VIP[01] nodes ARM: dts: r8a73a4: Add missing CMT1 interrupts arm64: dts: rockchip: Rename dwc3 device nodes on rk3399 to make dtc happy arm64: dts: rockchip: Replace RK805 PMIC node name with "pmic" on rk3328 boards clk: Unlink clock if failed to prepare or enable Revert "ALSA: hda/realtek: Fix pop noise on ALC225" usb: gadget: legacy: fix error return code in cdc_bind() usb: gadget: legacy: fix error return code in gncm_bind() usb: gadget: audio: Fix a missing error return value in audio_bind() usb: gadget: net2272: Fix a memory leak in an error handling path in 'net2272_plat_probe()' dwc3: Remove check for HWO flag in dwc3_gadget_ep_reclaim_trb_sg() clk: rockchip: fix incorrect configuration of rk3228 aclk_gpu* clocks exec: Move would_dump into flush_old_exec x86/unwind/orc: Fix error handling in __unwind_start() x86: Fix early boot crash on gcc-10, third try cifs: fix leaked reference on requeued write ARM: dts: imx27-phytec-phycard-s-rdk: Fix the I2C1 pinctrl entries ARM: dts: dra7: Fix bus_dma_limit for PCIe usb: xhci: Fix NULL pointer dereference when enqueuing trbs from urb sg list USB: gadget: fix illegal array access in binding with UDC usb: host: xhci-plat: keep runtime active when removing host usb: core: hub: limit HUB_QUIRK_DISABLE_AUTOSUSPEND to USB5534B ALSA: usb-audio: Add control message quirk delay for Kingston HyperX headset ALSA: rawmidi: Fix racy buffer resize under concurrent accesses ALSA: hda/realtek - Limit int mic boost for Thinkpad T530 gcc-10: avoid shadowing standard library 'free()' in crypto gcc-10: disable 'restrict' warning for now gcc-10: disable 'stringop-overflow' warning for now gcc-10: disable 'array-bounds' warning for now gcc-10: disable 'zero-length-bounds' warning for now Stop the ad-hoc games with -Wno-maybe-initialized kbuild: compute false-positive -Wmaybe-uninitialized cases in Kconfig gcc-10 warnings: fix low-hanging fruit pnp: Use list_for_each_entry() instead of open coding hwmon: (da9052) Synchronize access with mfd IB/mlx4: Test return value of calls to ib_get_cached_pkey netfilter: nft_set_rbtree: Introduce and use nft_rbtree_interval_start() arm64: fix the flush_icache_range arguments in machine_kexec netfilter: conntrack: avoid gcc-10 zero-length-bounds warning NFSv4: Fix fscache cookie aux_data to ensure change_attr is included nfs: fscache: use timespec64 in inode auxdata NFS: Fix fscache super_cookie index_key from changing after umount mmc: block: Fix request completion in the CQE timeout path mmc: core: Check request type before completing the request i40iw: Fix error handling in i40iw_manage_arp_cache() pinctrl: cherryview: Add missing spinlock usage in chv_gpio_irq_handler pinctrl: baytrail: Enable pin configuration setting for GPIO chip gfs2: Another gfs2_walk_metadata fix ALSA: hda/realtek - Fix S3 pop noise on Dell Wyse ipc/util.c: sysvipc_find_ipc() incorrectly updates position index drm/qxl: lost qxl_bo_kunmap_atomic_page in qxl_image_init_helper() ALSA: hda/hdmi: fix race in monitor detection during probe cpufreq: intel_pstate: Only mention the BIOS disabling turbo mode once dmaengine: mmp_tdma: Reset channel error on release dmaengine: pch_dma.c: Avoid data race between probe and irq handler riscv: fix vdso build with lld tcp: fix SO_RCVLOWAT hangs with fat skbs net: tcp: fix rx timestamp behavior for tcp_recvmsg netprio_cgroup: Fix unlimited memory leak of v2 cgroups net: ipv4: really enforce backoff for redirects net: dsa: loop: Add module soft dependency hinic: fix a bug of ndo_stop virtio_net: fix lockdep warning on 32 bit tcp: fix error recovery in tcp_zerocopy_receive() Revert "ipv6: add mtu lock check in __ip6_rt_update_pmtu" pppoe: only process PADT targeted at local interfaces net: phy: fix aneg restart in phy_ethtool_set_eee netlabel: cope with NULL catmap net: fix a potential recursive NETDEV_FEAT_CHANGE mmc: sdhci-acpi: Add SDHCI_QUIRK2_BROKEN_64_BIT_DMA for AMDI0040 scsi: sg: add sg_remove_request in sg_write virtio-blk: handle block_device_operations callbacks after hot unplug drop_monitor: work around gcc-10 stringop-overflow warning net: moxa: Fix a potential double 'free_irq()' net/sonic: Fix a resource leak in an error handling path in 'jazz_sonic_probe()' shmem: fix possible deadlocks on shmlock_user_lock net: dsa: Do not make user port errors fatal ANDROID: rtc: class: call hctosys in resource managed registration ANDROID: GKI: Update the ABI xml and whitelist ANDROID: power_supply: Add RTX power-supply property f2fs: flush dirty meta pages when flushing them f2fs: fix checkpoint=disable:%u%% f2fs: rework filename handling f2fs: split f2fs_d_compare() from f2fs_match_name() f2fs: don't leak filename in f2fs_try_convert_inline_dir() ANDROID: clang: update to 11.0.1 FROMLIST: x86_64: fix jiffies ODR violation ANDROID: arm64: vdso: Fix removing SCS flags ANDROID: GKI: Update the ABI xml and whitelist ANDROID: Incremental fs: wake up log pollers less often ANDROID: Incremental fs: Fix scheduling while atomic error ANDROID: Incremental fs: Avoid continually recalculating hashes ANDROID: export: Disable symbol trimming on modules ANDROID: GKI: Update the ABI xml and whitelist ANDROID: fscrypt: set dun_bytes more precisely ANDROID: dm-default-key: set dun_bytes more precisely ANDROID: block: backport the ability to specify max_dun_bytes ANDROID: Revert "ANDROID: GKI: gki_defconfig: CONFIG_DM_DEFAULT_KEY=m" Linux 4.19.123 ipc/mqueue.c: change __do_notify() to bypass check_kill_permission() scripts/decodecode: fix trapping instruction formatting objtool: Fix stack offset tracking for indirect CFAs netfilter: nf_osf: avoid passing pointer to local var netfilter: nat: never update the UDP checksum when it's 0 x86/unwind/orc: Fix premature unwind stoppage due to IRET frames x86/unwind/orc: Fix error path for bad ORC entry type x86/unwind/orc: Prevent unwinding before ORC initialization x86/unwind/orc: Don't skip the first frame for inactive tasks x86/entry/64: Fix unwind hints in rewind_stack_do_exit() x86/entry/64: Fix unwind hints in kernel exit path x86/entry/64: Fix unwind hints in register clearing code batman-adv: Fix refcnt leak in batadv_v_ogm_process batman-adv: Fix refcnt leak in batadv_store_throughput_override batman-adv: Fix refcnt leak in batadv_show_throughput_override batman-adv: fix batadv_nc_random_weight_tq KVM: VMX: Mark RCX, RDX and RSI as clobbered in vmx_vcpu_run()'s asm blob KVM: VMX: Explicitly reference RCX as the vmx_vcpu pointer in asm blobs coredump: fix crash when umh is disabled staging: gasket: Check the return value of gasket_get_bar_index() mm/page_alloc: fix watchdog soft lockups during set_zone_contiguous() arm64: hugetlb: avoid potential NULL dereference KVM: arm64: Fix 32bit PC wrap-around KVM: arm: vgic: Fix limit condition when writing to GICD_I[CS]ACTIVER tracing: Add a vmalloc_sync_mappings() for safe measure USB: serial: garmin_gps: add sanity checking for data length USB: uas: add quirk for LaCie 2Big Quadra HID: usbhid: Fix race between usbhid_close() and usbhid_stop() sctp: Fix bundling of SHUTDOWN with COOKIE-ACK HID: wacom: Read HID_DG_CONTACTMAX directly for non-generic devices net: stricter validation of untrusted gso packets bnxt_en: Fix VF anti-spoof filter setup. bnxt_en: Improve AER slot reset. net/mlx5: Fix command entry leak in Internal Error State net/mlx5: Fix forced completion access non initialized command entry bnxt_en: Fix VLAN acceleration handling in bnxt_fix_features(). tipc: fix partial topology connection closure sch_sfq: validate silly quantum values sch_choke: avoid potential panic in choke_reset() net: usb: qmi_wwan: add support for DW5816e net_sched: sch_skbprio: add message validation to skbprio_change() net/mlx4_core: Fix use of ENOSPC around mlx4_counter_alloc() net: macsec: preserve ingress frame ordering fq_codel: fix TCA_FQ_CODEL_DROP_BATCH_SIZE sanity checks dp83640: reverse arguments to list_add_tail vt: fix unicode console freeing with a common interface tracing/kprobes: Fix a double initialization typo USB: serial: qcserial: Add DW5816e support ANDROID: usb: gadget: Add missing inline qualifier to stub functions ANDROID: Drop ABI monitoring from KASAN build config ANDROID: Rename build.config.gki.arch_kasan ANDROID: GKI: Enable CONFIG_STATIC_USERMODEHELPER ANDROID: dm-default-key: Update key size for wrapped keys ANDROID: gki_defconfig: enable CONFIG_MMC_CRYPTO ANDROID: mmc: MMC crypto API ANDROID: GKI: Update the ABI xml and whitelist ANDROID: GKI: add missing exports for cam_smmu_api.ko Linux 4.19.122 drm/atomic: Take the atomic toys away from X cgroup, netclassid: remove double cond_resched mac80211: add ieee80211_is_any_nullfunc() platform/x86: GPD pocket fan: Fix error message when temp-limits are out of range ALSA: hda: Match both PCI ID and SSID for driver blacklist hexagon: define ioremap_uc hexagon: clean up ioremap mfd: intel-lpss: Use devm_ioremap_uc for MMIO lib: devres: add a helper function for ioremap_uc drm/amdgpu: Fix oops when pp_funcs is unset in ACPI event sctp: Fix SHUTDOWN CTSN Ack in the peer restart case net: systemport: suppress warnings on failed Rx SKB allocations net: bcmgenet: suppress warnings on failed Rx SKB allocations lib/mpi: Fix building for powerpc with clang scripts/config: allow colons in option strings for sed s390/ftrace: fix potential crashes when switching tracers cifs: protect updating server->dstaddr with a spinlock ASoC: rsnd: Fix "status check failed" spam for multi-SSI ASoC: rsnd: Don't treat master SSI in multi SSI setup as parent net: stmmac: Fix sub-second increment net: stmmac: fix enabling socfpga's ptp_ref_clock wimax/i2400m: Fix potential urb refcnt leak drm/amdgpu: Correctly initialize thermal controller for GPUs with Powerplay table v0 (e.g Hawaii) ASoC: codecs: hdac_hdmi: Fix incorrect use of list_for_each_entry ASoC: rsnd: Fix HDMI channel mapping for multi-SSI mode ASoC: rsnd: Fix parent SSI start/stop in multi-SSI mode usb: dwc3: gadget: Properly set maxpacket limit ASoC: sgtl5000: Fix VAG power-on handling selftests/ipc: Fix test failure seen after initial test run ASoC: topology: Check return value of pcm_new_ver powerpc/pci/of: Parse unassigned resources vhost: vsock: kick send_pkt worker once device is started ANDROID: GKI: fix build warning on 32bits due to ASoC msm change ANDROID: GKI: fix build error on 32bits due to ASoC msm change ANDROID: GKI: update abi definition due to FAIR_GROUP_SCHED removal ANDROID: GKI: Remove FAIR_GROUP_SCHED ANDROID: GKI: BULK update ABI XML representation and qcom whitelist ANDROID: build.config.gki.aarch64: Enable WHITELIST_STRICT_MODE ANDROID: GKI: Update the ABI xml and qcom whitelist ANDROID: remove unused variable ANDROID: Drop ABI monitoring from KASAN build config Linux 4.19.121 mmc: meson-mx-sdio: remove the broken ->card_busy() op mmc: meson-mx-sdio: Set MMC_CAP_WAIT_WHILE_BUSY mmc: sdhci-msm: Enable host capabilities pertains to R1b response mmc: sdhci-pci: Fix eMMC driver strength for BYT-based controllers mmc: sdhci-xenon: fix annoying 1.8V regulator warning mmc: cqhci: Avoid false "cqhci: CQE stuck on" by not open-coding timeout loop btrfs: transaction: Avoid deadlock due to bad initialization timing of fs_info::journal_info btrfs: fix partial loss of prealloc extent past i_size after fsync selinux: properly handle multiple messages in selinux_netlink_send() dmaengine: dmatest: Fix iteration non-stop logic nfs: Fix potential posix_acl refcnt leak in nfs3_set_acl ALSA: opti9xx: shut up gcc-10 range warning iommu/amd: Fix legacy interrupt remapping for x2APIC-enabled system scsi: target/iblock: fix WRITE SAME zeroing iommu/qcom: Fix local_base status check vfio/type1: Fix VA->PA translation for PFNMAP VMAs in vaddr_get_pfn() vfio: avoid possible overflow in vfio_iommu_type1_pin_pages RDMA/core: Fix race between destroy and release FD object RDMA/core: Prevent mixed use of FDs between shared ufiles RDMA/mlx4: Initialize ib_spec on the stack RDMA/mlx5: Set GRH fields in query QP on RoCE scsi: qla2xxx: check UNLOADING before posting async work scsi: qla2xxx: set UNLOADING before waiting for session deletion dm multipath: use updated MPATHF_QUEUE_IO on mapping for bio-based mpath dm writecache: fix data corruption when reloading the target dm verity fec: fix hash block number in verity_fec_decode PM: hibernate: Freeze kernel threads in software_resume() PM: ACPI: Output correct message on target power state ALSA: pcm: oss: Place the plugin buffer overflow checks correctly ALSA: hda/hdmi: fix without unlocked before return ALSA: usb-audio: Correct a typo of NuPrime DAC-10 USB ID ALSA: hda/realtek - Two front mics on a Lenovo ThinkCenter btrfs: fix block group leak when removing fails drm/qxl: qxl_release use after free drm/qxl: qxl_release leak in qxl_hw_surface_alloc() drm/qxl: qxl_release leak in qxl_draw_dirty_fb() drm/edid: Fix off-by-one in DispID DTD pixel clock ANDROID: GKI: Bulk update ABI XML representation ANDROID: GKI: Enable net testing options ANDROID: gki_defconfig: Enable CONFIG_REMOTEPROC ANDROID: Rename build.config.gki.arch_kasan ANDROID: GKI: Update ABI for IOMMU ANDROID: Incremental fs: Fix issues with very large files ANDROID: Correct build.config branch name ANDROID: GKI: Bulk update ABI XML representation and whitelist. UPSTREAM: vdso: Fix clocksource.h macro detection ANDROID: GKI: update abi definition due to added padding ANDROID: GKI: networking: add Android ABI padding to a lot of networking structures ANDROID: GKI: dma-mapping.h: add Android ABI padding to a structure ANDROID: GKI: ioport.h: add Android ABI padding to a structure ANDROID: GKI: iomap.h: add Android ABI padding to a structure ANDROID: GKI: genhd.h: add Android ABI padding to some structures ANDROID: GKI: hrtimer.h: add Android ABI padding to a structure ANDROID: GKI: ethtool.h: add Android ABI padding to a structure ANDROID: GKI: sched: add Android ABI padding to some structures ANDROID: GKI: kernfs.h: add Android ABI padding to some structures ANDROID: GKI: kobject.h: add Android ABI padding to some structures ANDROID: GKI: mm.h: add Android ABI padding to a structure ANDROID: GKI: mmu_notifier.h: add Android ABI padding to some structures ANDROID: GKI: pci: add Android ABI padding to some structures ANDROID: GKI: irqdomain.h: add Android ABI padding to a structure ANDROID: GKI: blk_types.h: add Android ABI padding to a structure ANDROID: GKI: scsi.h: add Android ABI padding to a structure ANDROID: GKI: quota.h: add Android ABI padding to some structures ANDROID: GKI: timer.h: add Android ABI padding to a structure ANDROID: GKI: user_namespace.h: add Android ABI padding to a structure FROMGIT: f2fs: fix missing check for f2fs_unlock_op Linux 4.19.120 propagate_one(): mnt_set_mountpoint() needs mount_lock ext4: check for non-zero journal inum in ext4_calculate_overhead qed: Fix use after free in qed_chain_free bpf, x86_32: Fix clobbering of dst for BPF_JSET hwmon: (jc42) Fix name to have no illegal characters ext4: convert BUG_ON's to WARN_ON's in mballoc.c ext4: increase wait time needed before reuse of deleted inode numbers ext4: use matching invalidatepage in ext4_writepage arm64: Delete the space separator in __emit_inst ALSA: hda: call runtime_allow() for all hda controllers xen/xenbus: ensure xenbus_map_ring_valloc() returns proper grant status objtool: Support Clang non-section symbols in ORC dump objtool: Fix CONFIG_UBSAN_TRAP unreachable warnings scsi: target: tcmu: reset_ring should reset TCMU_DEV_BIT_BROKEN scsi: target: fix PR IN / READ FULL STATUS for FC ALSA: hda: Explicitly permit using autosuspend if runtime PM is supported ALSA: hda: Keep the controller initialization even if no codecs found xfs: fix partially uninitialized structure in xfs_reflink_remap_extent x86: hyperv: report value of misc_features net: fec: set GPR bit on suspend by DT configuration. bpf, x86: Fix encoding for lower 8-bit registers in BPF_STX BPF_B xfs: clear PF_MEMALLOC before exiting xfsaild thread mm: shmem: disable interrupt when acquiring info->lock in userfaultfd_copy path bpf, x86_32: Fix incorrect encoding in BPF_LDX zero-extension perf/core: fix parent pid/tid in task exit events net/mlx5: Fix failing fw tracer allocation on s390 cpumap: Avoid warning when CONFIG_DEBUG_PER_CPU_MAPS is enabled ARM: dts: bcm283x: Disable dsi0 node PCI: Move Apex Edge TPU class quirk to fix BAR assignment PCI: Avoid ASMedia XHCI USB PME# from D0 defect svcrdma: Fix leak of svc_rdma_recv_ctxt objects svcrdma: Fix trace point use-after-free race xfs: acquire superblock freeze protection on eofblocks scans net/cxgb4: Check the return from t4_query_params properly rxrpc: Fix DATA Tx to disable nofrag for UDP on AF_INET6 socket i2c: altera: use proper variable to hold errno nfsd: memory corruption in nfsd4_lock() ASoC: wm8960: Fix wrong clock after suspend & resume ASoC: tas571x: disable regulators on failed probe ASoC: q6dsp6: q6afe-dai: add missing channels to MI2S DAIs iio:ad7797: Use correct attribute_group usb: gadget: udc: bdc: Remove unnecessary NULL checks in bdc_req_complete usb: dwc3: gadget: Do link recovery for SS and SSP binder: take read mode of mmap_sem in binder_alloc_free_page() include/uapi/linux/swab.h: fix userspace breakage, use __BITS_PER_LONG for swap mtd: cfi: fix deadloop in cfi_cmdset_0002.c do_write_buffer remoteproc: Fix wrong rvring index computation FROMLIST: PM / devfreq: Restart previous governor if new governor fails to start ANDROID: GKI: arm64: Enable GZIP and LZ4 kernel compression modes ANDROID: GKI: arm64: gki_defconfig: Set arm_smmu configuration ANDROID: GKI: iommu/arm-smmu: Modularize ARM SMMU driver ANDROID: GKI: iommu: Snapshot of vendor changes ANDROID: GKI: Additions to ARM SMMU register definitions ANDROID: GKI: iommu/io-pgtable-arm: LPAE related updates by vendor ANDROID: GKI: common: dma-mapping: make dma_common_contiguous_remap more robust ANDROID: GKI: dma-coherent: Expose device base address and size ANDROID: GKI: arm64: add support for NO_KERNEL_MAPPING and STRONGLY_ORDERED ANDROID: GKI: dma-mapping: Add dma_remap functions ANDROID: GKI: arm64: Support early fixup for CMA ANDROID: GKI: iommu: dma-mapping-fast: Fast ARMv7/v8 Long Descriptor Format ANDROID: GKI: arm64: dma-mapping: add support for IOMMU mapper ANDROID: GKI: add ARCH_NR_GPIO for ABI match ANDROID: GKI: kernel: Export symbol of `cpu_do_idle` ANDROID: GKI: kernel: Export symbols needed by msm_minidump.ko and minidump_log.ko (again) ANDROID: GKI: add missing exports for __flush_dcache_area ANDROID: GKI: arm64: Export caching APIs ANDROID: GKI: arm64: provide dma cache routines with same API as 32 bit ANDROID: gki_defconfig: add FORTIFY_SOURCE, remove SPMI_MSM_PMIC_ARB Revert "ANDROID: GKI: spmi: pmic-arb: don't enable SPMI_MSM_PMIC_ARB by default" ANDROID: GKI: update abi definitions after adding padding ANDROID: GKI: elevator: add Android ABI padding to some structures ANDROID: GKI: dentry: add Android ABI padding to some structures ANDROID: GKI: bio: add Android ABI padding to some structures ANDROID: GKI: scsi: add Android ABI padding to some structures ANDROID: GKI: ufs: add Android ABI padding to some structures ANDROID: GKI: workqueue.h: add Android ABI padding to some structures ANDROID: GKI: fs.h: add Android ABI padding to some structures ANDROID: GKI: USB: add Android ABI padding to some structures ANDROID: GKI: mm: add Android ABI padding to some structures ANDROID: GKI: mount.h: add Android ABI padding to some structures ANDROID: GKI: sched.h: add Android ABI padding to some structures ANDROID: GKI: sock.h: add Android ABI padding to some structures ANDROID: GKI: module.h: add Android ABI padding to some structures ANDROID: GKI: device.h: add Android ABI padding to some structures ANDROID: GKI: phy: add Android ABI padding to some structures ANDROID: GKI: add android_kabi.h ANDROID: ABI: update due to previous changes in the tree BACKPORT: sched/core: Fix reset-on-fork from RT with uclamp ANDROID: GKI: Add support for missing V4L2 symbols ANDROID: GKI: Bulk update ABI XML representation ANDROID: GKI: arm64: psci: Support for OS initiated scheme ANDROID: GKI: net: add counter for number of frames coalesced in GRO ANDROID: GKI: cfg80211: Include length of kek in rekey data BACKPORT: loop: change queue block size to match when using DIO ANDROID: Incremental fs: Add setattr call ANDROID: GKI: enable CONFIG_RTC_SYSTOHC ANDROID: GKI: ipv4: add vendor padding to __IPV4_DEVCONF_* enums Revert "ANDROID: GKI: ipv4: increase __IPV4_DEVCONF_MAX to 64" ANDROID: driver: gpu: drm: fix export symbol types ANDROID: SoC: core: fix export symbol type ANDROID: ufshcd-crypto: fix export symbol type ANDROID: GKI: drivers: mailbox: fix race resulting in multiple message submission ANDROID: GKI: arm64: gki_defconfig: Enable a few thermal configs Revert "ANDROID: GKI: add base.h include to match MODULE_VERSIONS" FROMLIST: thermal: Make cooling device trip point writable from sysfs ANDROID: GKI: drivers: thermal: cpu_cooling: Use CPU ID as cooling device ID ANDROID: GKI: PM / devfreq: Allow min freq to be 0 ANDROID: GKI: arm64: gki_defconfig: Enable REGULATOR_PROXY_CONSUMER ANDROID: GKI: Bulk Update ABI XML representation ANDROID: KASAN support for GKI remove CONFIG_CC_WERROR ANDROID: KASAN support for GKI ANDROID: virt_wifi: fix export symbol types ANDROID: vfs: fix export symbol type ANDROID: vfs: fix export symbol types ANDROID: fscrypt: fix export symbol type ANDROID: cfi: fix export symbol types ANDROID: bpf: fix export symbol type Linux 4.19.119 s390/mm: fix page table upgrade vs 2ndary address mode accesses xfs: Fix deadlock between AGI and AGF with RENAME_WHITEOUT serial: sh-sci: Make sure status register SCxSR is read in correct sequence xhci: prevent bus suspend if a roothub port detected a over-current condition usb: f_fs: Clear OS Extended descriptor counts to zero in ffs_data_reset() usb: dwc3: gadget: Fix request completion check UAS: fix deadlock in error handling and PM flushing work UAS: no use logging any details in case of ENODEV cdc-acm: introduce a cool down cdc-acm: close race betrween suspend() and acm_softint staging: vt6656: Power save stop wake_up_count wrap around. staging: vt6656: Fix pairwise key entry save. staging: vt6656: Fix drivers TBTT timing counter. staging: vt6656: Fix calling conditions of vnt_set_bss_mode staging: vt6656: Don't set RCR_MULTICAST or RCR_BROADCAST by default. vt: don't use kmalloc() for the unicode screen buffer vt: don't hardcode the mem allocation upper bound staging: comedi: Fix comedi_device refcnt leak in comedi_open staging: comedi: dt2815: fix writing hi byte of analog output powerpc/setup_64: Set cache-line-size based on cache-block-size ARM: imx: provide v7_cpu_resume() only on ARM_CPU_SUSPEND=y iwlwifi: mvm: beacon statistics shouldn't go backwards iwlwifi: pcie: actually release queue memory in TVQM ASoC: dapm: fixup dapm kcontrol widget audit: check the length of userspace generated audit records usb-storage: Add unusual_devs entry for JMicron JMS566 tty: rocket, avoid OOB access tty: hvc: fix buffer overflow during hvc_alloc(). KVM: VMX: Enable machine check support for 32bit targets KVM: Check validity of resolved slot when searching memslots KVM: s390: Return last valid slot if approx index is out-of-bounds tpm: ibmvtpm: retry on H_CLOSED in tpm_ibmvtpm_send() tpm/tpm_tis: Free IRQ if probing fails ALSA: usb-audio: Filter out unsupported sample rates on Focusrite devices ALSA: usb-audio: Fix usb audio refcnt leak when getting spdif ALSA: hda/realtek - Add new codec supported for ALC245 ALSA: hda/realtek - Fix unexpected init_amp override ALSA: usx2y: Fix potential NULL dereference tools/vm: fix cross-compile build mm/ksm: fix NULL pointer dereference when KSM zero page is enabled mm/hugetlb: fix a addressing exception caused by huge_pte_offset vmalloc: fix remap_vmalloc_range() bounds checks USB: hub: Fix handling of connect changes during sleep USB: core: Fix free-while-in-use bug in the USB S-Glibrary USB: early: Handle AMD's spec-compliant identifiers, too USB: Add USB_QUIRK_DELAY_CTRL_MSG and USB_QUIRK_DELAY_INIT for Corsair K70 RGB RAPIDFIRE USB: sisusbvga: Change port variable from signed to unsigned fs/namespace.c: fix mountpoint reference counter race iio: xilinx-xadc: Make sure not exceed maximum samplerate iio: xilinx-xadc: Fix sequencer configuration for aux channels in simultaneous mode iio: xilinx-xadc: Fix clearing interrupt when enabling trigger iio: xilinx-xadc: Fix ADC-B powerdown iio: adc: stm32-adc: fix sleep in atomic context iio: st_sensors: rely on odr mask to know if odr can be set iio: core: remove extra semi-colon from devm_iio_device_register() macro ALSA: usb-audio: Add connector notifier delegation ALSA: usb-audio: Add static mapping table for ALC1220-VB-based mobos ALSA: hda: Remove ASUS ROG Zenith from the blacklist KEYS: Avoid false positive ENOMEM error on key read mlxsw: Fix some IS_ERR() vs NULL bugs vrf: Check skb for XFRM_TRANSFORMED flag xfrm: Always set XFRM_TRANSFORMED in xfrm{4,6}_output_finish net: dsa: b53: b53_arl_rw_op() needs to select IVL or SVL net: dsa: b53: Rework ARL bin logic net: dsa: b53: Fix ARL register definitions net: dsa: b53: Lookup VID in ARL searches when VLAN is enabled vrf: Fix IPv6 with qdisc and xfrm team: fix hang in team_mode_get() tcp: cache line align MAX_TCP_HEADER sched: etf: do not assume all sockets are full blown net/x25: Fix x25_neigh refcnt leak when receiving frame net: stmmac: dwmac-meson8b: Add missing boundary to RGMII TX clock array net: netrom: Fix potential nr_neigh refcnt leak in nr_add_node net: bcmgenet: correct per TX/RX ring statistics macvlan: fix null dereference in macvlan_device_event() macsec: avoid to set wrong mtu ipv6: fix restrict IPV6_ADDRFORM operation cxgb4: fix large delays in PTP synchronization cxgb4: fix adapter crash due to wrong MC size x86/KVM: Clean up host's steal time structure x86/KVM: Make sure KVM_VCPU_FLUSH_TLB flag is not missed x86/kvm: Cache gfn to pfn translation x86/kvm: Introduce kvm_(un)map_gfn() KVM: Properly check if "page" is valid in kvm_vcpu_unmap kvm: fix compile on s390 part 2 kvm: fix compilation on s390 kvm: fix compilation on aarch64 KVM: Introduce a new guest mapping API KVM: nVMX: Always sync GUEST_BNDCFGS when it comes from vmcs01 KVM: VMX: Zero out *all* general purpose registers after VM-Exit f2fs: fix to avoid memory leakage in f2fs_listxattr blktrace: fix dereference after null check blktrace: Protect q->blk_trace with RCU net: ipv6_stub: use ip6_dst_lookup_flow instead of ip6_dst_lookup net: ipv6: add net argument to ip6_dst_lookup_flow PCI/ASPM: Allow re-enabling Clock PM scsi: smartpqi: fix call trace in device discovery virtio-blk: improve virtqueue error to BLK_STS tracing/selftests: Turn off timeout setting drm/amd/display: Not doing optimize bandwidth if flip pending. xhci: Ensure link state is U3 after setting USB_SS_PORT_LS_U3 ASoC: Intel: bytcr_rt5640: Add quirk for MPMAN MPWIN895CL tablet perf/core: Disable page faults when getting phys address pwm: bcm2835: Dynamically allocate base pwm: renesas-tpu: Fix late Runtime PM enablement Revert "powerpc/64: irq_work avoid interrupt when called with hardware irqs enabled" loop: Better discard support for block devices s390/cio: avoid duplicated 'ADD' uevents kconfig: qconf: Fix a few alignment issues ipc/util.c: sysvipc_find_ipc() should increase position index selftests: kmod: fix handling test numbers above 9 kernel/gcov/fs.c: gcov_seq_next() should increase position index nvme: fix deadlock caused by ANA update wrong locking ASoC: Intel: atom: Take the drv->lock mutex before calling sst_send_slot_map() scsi: iscsi: Report unbind session event when the target has been removed pwm: rcar: Fix late Runtime PM enablement ceph: don't skip updating wanted caps when cap is stale ceph: return ceph_mdsc_do_request() errors from __get_parent() scsi: lpfc: Fix crash in target side cable pulls hitting WAIT_FOR_UNREG scsi: lpfc: Fix kasan slab-out-of-bounds error in lpfc_unreg_login watchdog: reset last_hw_keepalive time at start arm64: Silence clang warning on mismatched value/register sizes arm64: compat: Workaround Neoverse-N1 #1542419 for compat user-space arm64: Fake the IminLine size on systems affected by Neoverse-N1 #1542419 arm64: errata: Hide CTR_EL0.DIC on systems affected by Neoverse-N1 #1542419 arm64: Add part number for Neoverse N1 vti4: removed duplicate log message. crypto: mxs-dcp - make symbols 'sha1_null_hash' and 'sha256_null_hash' static bpftool: Fix printing incorrect pointer in btf_dump_ptr drm/msm: Use the correct dma_sync calls harder ext4: fix extent_status fragmentation for plain files ANDROID: abi_gki_aarch64_cuttlefish_whitelist: remove stale symbols ANDROID: GKI: ipv4: increase __IPV4_DEVCONF_MAX to 64 ANDROID: GKI: power: add missing export for POWER_RESET_QCOM=m BACKPORT: cfg80211: Support key configuration for Beacon protection (BIGTK) BACKPORT: cfg80211: Enhance the AKM advertizement to support per interface. UPSTREAM: sysrq: Use panic() to force a crash ANDROID: GKI: kernel: sound: update codec options with block size ANDROID: add compat cross compiler ANDROID: x86/vdso: disable LTO only for VDSO BACKPORT: arm64: vdso32: Enable Clang Compilation UPSTREAM: arm64: compat: vdso: Expose BUILD_VDSO32 BACKPORT: lib/vdso: Enable common headers BACKPORT: arm: vdso: Enable arm to use common headers BACKPORT: x86/vdso: Enable x86 to use common headers BACKPORT: mips: vdso: Enable mips to use common headers UPSTREAM: arm64: vdso32: Include common headers in the vdso library UPSTREAM: arm64: vdso: Include common headers in the vdso library UPSTREAM: arm64: Introduce asm/vdso/processor.h BACKPORT: arm64: vdso32: Code clean up UPSTREAM: linux/elfnote.h: Replace elf.h with UAPI equivalent UPSTREAM: scripts: Fix the inclusion order in modpost UPSTREAM: common: Introduce processor.h UPSTREAM: linux/ktime.h: Extract common header for vDSO UPSTREAM: linux/jiffies.h: Extract common header for vDSO UPSTREAM: linux/time64.h: Extract common header for vDSO BACKPORT: linux/time32.h: Extract common header for vDSO BACKPORT: linux/time.h: Extract common header for vDSO UPSTREAM: linux/math64.h: Extract common header for vDSO BACKPORT: linux/clocksource.h: Extract common header for vDSO BACKPORT: mips: Introduce asm/vdso/clocksource.h BACKPORT: arm64: Introduce asm/vdso/clocksource.h BACKPORT: arm: Introduce asm/vdso/clocksource.h BACKPORT: x86: Introduce asm/vdso/clocksource.h UPSTREAM: linux/limits.h: Extract common header for vDSO BACKPORT: linux/kernel.h: split *_MAX and *_MIN macros into <linux/limits.h> BACKPORT: linux/bits.h: Extract common header for vDSO UPSTREAM: linux/const.h: Extract common header for vDSO BACKPORT: arm64: vdso: fix flip/flop vdso build bug UPSTREAM: lib/vdso: Allow the high resolution parts to be compiled out UPSTREAM: lib/vdso: Only read hrtimer_res when needed in __cvdso_clock_getres() UPSTREAM: lib/vdso: Mark do_hres() and do_coarse() as __always_inline UPSTREAM: lib/vdso: Avoid duplication in __cvdso_clock_getres() UPSTREAM: lib/vdso: Let do_coarse() return 0 to simplify the callsite UPSTREAM: lib/vdso: Remove checks on return value for 32 bit vDSO UPSTREAM: lib/vdso: Build 32 bit specific functions in the right context UPSTREAM: lib/vdso: Make __cvdso_clock_getres() static UPSTREAM: lib/vdso: Make clock_getres() POSIX compliant again UPSTREAM: lib/vdso/32: Provide legacy syscall fallbacks UPSTREAM: lib/vdso: Move fallback invocation to the callers UPSTREAM: lib/vdso/32: Remove inconsistent NULL pointer checks UPSTREAM: lib/vdso: Make delta calculation work correctly UPSTREAM: arm64: compat: Fix syscall number of compat_clock_getres BACKPORT: arm64: lse: Fix LSE atomics with LLVM UPSTREAM: mips: Fix gettimeofday() in the vdso library UPSTREAM: mips: vdso: Fix __arch_get_hw_counter() BACKPORT: arm64: Kconfig: Make CONFIG_COMPAT_VDSO a proper Kconfig option UPSTREAM: arm64: vdso32: Rename COMPATCC to CC_COMPAT UPSTREAM: arm64: vdso32: Pass '--target' option to clang via VDSO_CAFLAGS UPSTREAM: arm64: vdso32: Don't use KBUILD_CPPFLAGS unconditionally UPSTREAM: arm64: vdso32: Move definition of COMPATCC into vdso32/Makefile UPSTREAM: arm64: Default to building compat vDSO with clang when CONFIG_CC_IS_CLANG UPSTREAM: lib: vdso: Remove CROSS_COMPILE_COMPAT_VDSO UPSTREAM: arm64: vdso32: Remove jump label config option in Makefile UPSTREAM: arm64: vdso32: Detect binutils support for dmb ishld BACKPORT: arm64: vdso: Remove stale files from old assembly implementation UPSTREAM: arm64: vdso32: Fix broken compat vDSO build warnings UPSTREAM: mips: compat: vdso: Use legacy syscalls as fallback BACKPORT: arm64: Relax Documentation/arm64/tagged-pointers.rst BACKPORT: arm64: Add tagged-address-abi.rst to index.rst UPSTREAM: arm64: vdso: Fix Makefile regression UPSTREAM: mips: vdso: Fix flip/flop vdso building bug UPSTREAM: mips: vdso: Fix source path UPSTREAM: mips: Add clock_gettime64 entry point UPSTREAM: mips: Add clock_getres entry point BACKPORT: mips: Add support for generic vDSO BACKPORT: arm64: vdso: Explicitly add build-id option BACKPORT: arm64: vdso: use $(LD) instead of $(CC) to link VDSO BACKPORT: arm64: vdso: Cleanup Makefiles UPSTREAM: arm64: vdso: Fix population of AT_SYSINFO_EHDR for compat vdso UPSTREAM: arm64: vdso: Fix compilation with clang older than 8 UPSTREAM: arm64: compat: Fix __arch_get_hw_counter() implementation UPSTREAM: arm64: Fix __arch_get_hw_counter() implementation UPSTREAM: x86/vdso/32: Use 32bit syscall fallback UPSTREAM: x86/vdso: Fix flip/flop vdso build bug UPSTREAM: x86/vdso: Give the [ph]vclock_page declarations real types UPSTREAM: x86/vdso: Add clock_gettime64() entry point BACKPORT: x86/vdso: Add clock_getres() entry point BACKPORT: x86/vdso: Switch to generic vDSO implementation UPSTREAM: x86/segments: Introduce the 'CPUNODE' naming to better document the segment limit CPU/node NR trick UPSTREAM: x86/vdso: Initialize the CPU/node NR segment descriptor earlier UPSTREAM: x86/vdso: Introduce helper functions for CPU and node number UPSTREAM: x86/segments/64: Rename the GDT PER_CPU entry to CPU_NUMBER BACKPORT: arm64: vdso: Enable vDSO compat support UPSTREAM: arm64: compat: Get sigreturn trampolines from vDSO UPSTREAM: arm64: elf: VDSO code page discovery UPSTREAM: arm64: compat: VDSO setup for compat layer UPSTREAM: arm64: vdso: Refactor vDSO code BACKPORT: arm64: compat: Add vDSO UPSTREAM: arm64: compat: Generate asm offsets for signals UPSTREAM: arm64: compat: Expose signal related structures UPSTREAM: arm64: compat: Add missing syscall numbers BACKPORT: arm64: vdso: Substitute gettimeofday() with C implementation UPSTREAM: timekeeping: Provide a generic update_vsyscall() implementation UPSTREAM: lib/vdso: Add compat support UPSTREAM: lib/vdso: Provide generic VDSO implementation UPSTREAM: vdso: Define standardized vdso_datapage UPSTREAM: hrtimer: Split out hrtimer defines into separate header UPSTREAM: nds32: Fix vDSO clock_getres() UPSTREAM: arm64: compat: Reduce address limit for 64K pages BACKPORT: arm64: compat: Add KUSER_HELPERS config option UPSTREAM: arm64: compat: Refactor aarch32_alloc_vdso_pages() BACKPORT: arm64: compat: Split kuser32 UPSTREAM: arm64: compat: Alloc separate pages for vectors and sigpage ANDROID: GKI: Update ABI XML representation ANDROID: GKI: Enable GENERIC_IRQ_CHIP ANDROID: GKI: power_supply: Add FG_TYPE power-supply property ANDROID: GKI: mm: export mm_trace_rss_stat for modules to report RSS changes ANDROID: GKI: gki_defconfig: Enable CONFIG_LEDS_TRIGGER_TRANSIENT ANDROID: GKI: gki_defconfig: Enable CONFIG_CPU_FREQ_STAT ANDROID: GKI: arm64: gki_defconfig: Disable HW tracing features ANDROID: GKI: gki_defconfig: Enable CONFIG_I2C_CHARDEV ANDROID: Incremental fs: Use simple compression in log buffer ANDROID: GKI: usb: core: Add support to parse config summary capability descriptors ANDROID: GKI: Update ABI XML representation ANDROID: dm-bow: Fix not to skip trim at framented range ANDROID: Remove VLA from uid_sys_stats.c f2fs: fix missing check for f2fs_unlock_op ANDROID: fix wakeup reason findings UPSTREAM: cfg80211: fix and clean up cfg80211_gen_new_bssid() UPSTREAM: cfg80211: save multi-bssid properties UPSTREAM: cfg80211: make BSSID generation function inline UPSTREAM: cfg80211: parse multi-bssid only if HW supports it UPSTREAM: cfg80211: Move Multiple BSS info to struct cfg80211_bss to be visible UPSTREAM: cfg80211: Properly track transmitting and non-transmitting BSS UPSTREAM: cfg80211: use for_each_element() for multi-bssid parsing UPSTREAM: cfg80211: Parsing of Multiple BSSID information in scanning UPSTREAM: cfg80211/nl80211: Offload OWE processing to user space in AP mode ANDROID: GKI: cfg80211: Sync nl80211 commands/feature with upstream ANDROID: GKI: gki_defconfig: Enable FW_LOADER_USER_HELPER* ANDROID: GKI: arm64: gki_defconfig: Disable CONFIG_ARM64_TAGGED_ADDR_ABI ANDROID: GKI: gki_defconfig: CONFIG_CHR_DEV_SG=y ANDROID: GKI: gki_defconfig: CONFIG_DM_DEFAULT_KEY=m ANDROID: update the ABI xml representation ANDROID: init: GKI: enable hidden configs for GPU Linux 4.19.118 bpf: fix buggy r0 retval refinement for tracing helpers KEYS: Don't write out to userspace while holding key semaphore mtd: phram: fix a double free issue in error path mtd: lpddr: Fix a double free in probe() mtd: spinand: Explicitly use MTD_OPS_RAW to write the bad block marker to OOB locktorture: Print ratio of acquisitions, not failures tty: evh_bytechan: Fix out of bounds accesses iio: si1133: read 24-bit signed integer for measurement fbdev: potential information leak in do_fb_ioctl() net: dsa: bcm_sf2: Fix overflow checks f2fs: fix to wait all node page writeback iommu/amd: Fix the configuration of GCR3 table root pointer libnvdimm: Out of bounds read in __nd_ioctl() power: supply: axp288_fuel_gauge: Broaden vendor check for Intel Compute Sticks. ext2: fix debug reference to ext2_xattr_cache ext2: fix empty body warnings when -Wextra is used iommu/vt-d: Fix mm reference leak drm/vc4: Fix HDMI mode validation f2fs: fix NULL pointer dereference in f2fs_write_begin() NFS: Fix memory leaks in nfs_pageio_stop_mirroring() drm/amdkfd: kfree the wrong pointer x86: ACPI: fix CPU hotplug deadlock KVM: s390: vsie: Fix possible race when shadowing region 3 tables compiler.h: fix error in BUILD_BUG_ON() reporting percpu_counter: fix a data race at vm_committed_as include/linux/swapops.h: correct guards for non_swap_entry() cifs: Allocate encryption header through kmalloc um: ubd: Prevent buffer overrun on command completion ext4: do not commit super on read-only bdev s390/cpum_sf: Fix wrong page count in error message powerpc/maple: Fix declaration made after definition s390/cpuinfo: fix wrong output when CPU0 is offline NFS: direct.c: Fix memory leak of dreq when nfs_get_lock_context fails NFSv4/pnfs: Return valid stateids in nfs_layout_find_inode_by_stateid() rtc: 88pm860x: fix possible race condition soc: imx: gpc: fix power up sequencing clk: tegra: Fix Tegra PMC clock out parents power: supply: bq27xxx_battery: Silence deferred-probe error clk: at91: usb: continue if clk_hw_round_rate() return zero x86/Hyper-V: Report crash data in die() when panic_on_oops is set x86/Hyper-V: Report crash register data when sysctl_record_panic_msg is not set x86/Hyper-V: Trigger crash enlightenment only once during system crash. x86/Hyper-V: Free hv_panic_page when fail to register kmsg dump x86/Hyper-V: Unload vmbus channel in hv panic callback xsk: Add missing check on user supplied headroom size rbd: call rbd_dev_unprobe() after unwatching and flushing notifies rbd: avoid a deadlock on header_rwsem when flushing notifies video: fbdev: sis: Remove unnecessary parentheses and commented code lib/raid6: use vdupq_n_u8 to avoid endianness warnings x86/Hyper-V: Report crash register data or kmsg before running crash kernel of: overlay: kmemleak in dup_and_fixup_symbol_prop() of: unittest: kmemleak in of_unittest_overlay_high_level() of: unittest: kmemleak in of_unittest_platform_populate() of: unittest: kmemleak on changeset destroy ALSA: hda: Don't release card at firmware loading error irqchip/mbigen: Free msi_desc on device teardown netfilter: nf_tables: report EOPNOTSUPP on unsupported flags/object type ARM: dts: imx6: Use gpc for FEC interrupt controller to fix wake on LAN. arm, bpf: Fix bugs with ALU64 {RSH, ARSH} BPF_K shift by 0 watchdog: sp805: fix restart handler ext4: use non-movable memory for superblock readahead scsi: sg: add sg_remove_request in sg_common_write objtool: Fix switch table detection in .text.unlikely arm, bpf: Fix offset overflow for BPF_MEM BPF_DW ANDROID: GKI: Bulk update ABI report. ANDROID: GKI: qos: Register irq notify after adding the qos request ANDROID: GKI: Add dual role mode to usb_dr_modes array UPSTREAM: virtio-gpu api: comment feature flags ANDROID: arch:arm64: Increase kernel command line size ANDROID: GKI: Add special linux_banner_ptr for modules Revert "ANDROID: GKI: Make linux_banner a C pointer" ANDROID: GKI: PM / devfreq: Add new flag to do simple clock scaling ANDROID: GKI: Resolve ABI diff for struct snd_usb_audio ANDROID: GKI: Bulk update ABI ANDROID: GKI: Update the whitelist for qcom SoCs ANDROID: GKI: arm64: gki_defconfig: Set CONFIG_SCSI_UFSHCD=m ANDROID: GKI: scsi: add option to override the command timeout ANDROID: GKI: scsi: Adjust DBD setting in mode sense for caching mode page per LLD ANDROID: add ion_stat tracepoint to common kernel UPSTREAM: gpu/trace: add a gpu total memory usage tracepoint Linux 4.19.117 mm/vmalloc.c: move 'area->pages' after if statement wil6210: remove reset file from debugfs wil6210: make sure Rx ring sizes are correlated wil6210: add general initialization/size checks wil6210: ignore HALP ICR if already handled wil6210: check rx_buff_mgmt before accessing it x86/resctrl: Fix invalid attempt at removing the default resource group x86/resctrl: Preserve CDP enable over CPU hotplug x86/microcode/AMD: Increase microcode PATCH_MAX_SIZE scsi: target: fix hang when multiple threads try to destroy the same iscsi session scsi: target: remove boilerplate code kvm: x86: Host feature SSBD doesn't imply guest feature SPEC_CTRL_SSBD ext4: do not zeroout extents beyond i_disksize drm/amd/powerplay: force the trim of the mclk dpm_levels if OD is enabled usb: dwc3: gadget: Don't clear flags before transfer ended usb: dwc3: gadget: don't enable interrupt when disabling endpoint mac80211_hwsim: Use kstrndup() in place of kasprintf() btrfs: check commit root generation in should_ignore_root tracing: Fix the race between registering 'snapshot' event trigger and triggering 'snapshot' operation keys: Fix proc_keys_next to increase position index ALSA: usb-audio: Check mapping at creating connector controls, too ALSA: usb-audio: Don't create jack controls for PCM terminals ALSA: usb-audio: Don't override ignore_ctl_error value from the map ALSA: usb-audio: Filter error from connector kctl ops, too ASoC: Intel: mrfld: return error codes when an error occurs ASoC: Intel: mrfld: fix incorrect check on p->sink ext4: fix incorrect inodes per group in error message ext4: fix incorrect group count in ext4_fill_super error message pwm: pca9685: Fix PWM/GPIO inter-operation jbd2: improve comments about freeing data buffers whose page mapping is NULL scsi: ufs: Fix ufshcd_hold() caused scheduling while atomic ovl: fix value of i_ino for lower hardlink corner case net: dsa: mt7530: fix tagged frames pass-through in VLAN-unaware mode net: stmmac: dwmac-sunxi: Provide TX and RX fifo sizes net: revert default NAPI poll timeout to 2 jiffies net: qrtr: send msgs from local of same id as broadcast net: ipv6: do not consider routes via gateways for anycast address check net: ipv4: devinet: Fix crash when add/del multicast IP with autojoin hsr: check protocol version in hsr_newlink() amd-xgbe: Use __napi_schedule() in BH context ANDROID: GKI: drivers: of-thermal: Relate thermal zones using same sensor ANDROID: GKI: Bulk ABI update ANDROID: GKI: dma: Add set_dma_mask hook to struct dma_map_ops ANDROID: GKI: ABI update due to recent patches FROMLIST: drm/prime: add support for virtio exported objects FROMLIST: dma-buf: add support for virtio exported objects UPSTREAM: drm/virtio: module_param_named() requires linux/moduleparam.h UPSTREAM: drm/virtio: fix resource id creation race UPSTREAM: drm/virtio: make resource id workaround runtime switchable. BACKPORT: drm/virtio: Drop deprecated load/unload initialization ANDROID: GKI: Add DRM_TTM config to GKI ANDROID: Bulk update the ABI xml representation ANDROID: GKI: spmi: pmic-arb: don't enable SPMI_MSM_PMIC_ARB by default ANDROID: GKI: attribute page lock and waitqueue functions as sched ANDROID: GKI: extcon: Fix Add usage of blocking notifier chain ANDROID: GKI: USB: pd: Extcon fix for C current ANDROID: drm/dsi: Fix byte order of DCS set/get brightness ANDROID: GKI: mm: Export symbols to modularize CONFIG_MSM_DRM ANDROID: GKI: ALSA: compress: Add support to send codec specific data ANDROID: GKI: ALSA: Compress - dont use lock for all ioctls ANDROID: GKI: ASoC: msm: qdsp6v2: add support for AMR_WB_PLUS offload ANDROID: GKI: msm: dolby: MAT and THD audiocodec name modification ANDROID: GKI: asoc: msm: Add support for compressed perf mode ANDROID: GKI: msm: audio: support for gapless_pcm ANDROID: GKI: uapi: msm: dolby: Support for TrueHD and MAT decoders ANDROID: GKI: ASoC: msm: qdsp6v2: Add TrueHD HDMI compress pass-though ANDROID: GKI: ALSA: compress: Add APTX format support in ALSA ANDROID: GKI: msm: qdsp6v2: Add timestamp support for compress capture ANDROID: GKI: SoC: msm: Add support for meta data in compressed TX ANDROID: GKI: ALSA: compress: Add DSD format support for ALSA ANDROID: GKI: ASoC: msm: qdsp6v2: add support for ALAC and APE offload ANDROID: GKI: SoC: msm: Add compressed TX and passthrough support ANDROID: GKI: ASoC: msm: qdsp6v2: Add FLAC in compress offload path ANDROID: GKI: ASoC: msm: add support for different compressed formats ANDROID: GKI: ASoC: msm: Update the encode option and sample rate ANDROID: GKI: Enable CONFIG_SND_VERBOSE_PROCFS in gki_defconfig ANDROID: GKI: Add hidden CONFIG_SND_SOC_COMPRESS to gki_defconfig ANDROID: GKI: ALSA: pcm: add locks for accessing runtime resource ANDROID: GKI: Update ABI for DRM changes ANDROID: GKI: Add drm_dp_send_dpcd_{read,write} accessor functions ANDROID: GKI: drm: Add drm_dp_mst_get_max_sdp_streams_supported accessor function ANDROID: GKI: drm: Add drm_dp_mst_has_fec accessor function ANDROID: GKI: Add 'dsc_info' to struct drm_dp_mst_port ANDROID: GKI: usb: Add support to handle USB SMMU S1 address ANDROID: GKI: usb: Add helper APIs to return xhci phys addresses ANDROID: Add C protos for dma_buf/drm_prime get_uuid ANDROID: GKI: Make linux_banner a C pointer ANDROID: GKI: Add 'refresh_rate', 'id' to struct drm_panel_notifier ANDROID: GKI: Add 'i2c_mutex' to struct drm_dp_aux ANDROID: GKI: Add 'checksum' to struct drm_connector Revert "BACKPORT: drm: Add HDR source metadata property" Revert "BACKPORT: drm: Parse HDR metadata info from EDID" ANDROID: drm: Add DP colorspace property ANDROID: GKI: drm: Initialize display->hdmi when parsing vsdb ANDROID: drivers: gpu: drm: add support to batch commands ANDROID: ABI: update the qcom whitelist ANDROID: GKI: ARM64: smp: add vendor field pending_ipi ANDROID: gki_defconfig: enable msm serial early console ANDROID: serial: msm_geni_serial_console : Add Earlycon support ANDROID: GKI: serial: core: export uart_console_device f2fs: fix quota_sync failure due to f2fs_lock_op f2fs: support read iostat f2fs: Fix the accounting of dcc->undiscard_blks f2fs: fix to handle error path of f2fs_ra_meta_pages() f2fs: report the discard cmd errors properly f2fs: fix long latency due to discard during umount f2fs: add tracepoint for f2fs iostat f2fs: introduce sysfs/data_io_flag to attach REQ_META/FUA ANDROID: GKI: update abi definition due to previous changes in the tree Linux 4.19.116 efi/x86: Fix the deletion of variables in mixed mode mfd: dln2: Fix sanity checking for endpoints etnaviv: perfmon: fix total and idle HI cyleces readout misc: echo: Remove unnecessary parentheses and simplify check for zero powerpc/fsl_booke: Avoid creating duplicate tlb1 entry ftrace/kprobe: Show the maxactive number on kprobe_events drm: Remove PageReserved manipulation from drm_pci_alloc drm/dp_mst: Fix clearing payload state on topology disable Revert "drm/dp_mst: Remove VCPI while disabling topology mgr" crypto: ccree - only try to map auth tag if needed crypto: ccree - dec auth tag size from cryptlen map crypto: ccree - don't mangle the request assoclen crypto: ccree - zero out internal struct before use crypto: ccree - improve error handling crypto: caam - update xts sector size for large input length dm zoned: remove duplicate nr_rnd_zones increase in dmz_init_zone() btrfs: use nofs allocations for running delayed items powerpc: Make setjmp/longjmp signature standard powerpc: Add attributes for setjmp/longjmp scsi: mpt3sas: Fix kernel panic observed on soft HBA unplug powerpc/kprobes: Ignore traps that happened in real mode powerpc/xive: Use XIVE_BAD_IRQ instead of zero to catch non configured IPIs powerpc/hash64/devmap: Use H_PAGE_THP_HUGE when setting up huge devmap PTE entries powerpc/64/tm: Don't let userspace set regs->trap via sigreturn powerpc/powernv/idle: Restore AMR/UAMOR/AMOR after idle xen/blkfront: fix memory allocation flags in blkfront_setup_indirect() ipmi: fix hung processes in __get_guid() libata: Return correct status in sata_pmp_eh_recover_pm() when ATA_DFLAG_DETACH is set hfsplus: fix crash and filesystem corruption when deleting files cpufreq: powernv: Fix use-after-free kmod: make request_module() return an error when autoloading is disabled clk: ingenic/jz4770: Exit with error if CGU init failed Input: i8042 - add Acer Aspire 5738z to nomux list s390/diag: fix display of diagnose call statistics perf tools: Support Python 3.8+ in Makefile ocfs2: no need try to truncate file beyond i_size fs/filesystems.c: downgrade user-reachable WARN_ONCE() to pr_warn_once() ext4: fix a data race at inode->i_blocks NFS: Fix a page leak in nfs_destroy_unlinked_subrequests() powerpc/pseries: Avoid NULL pointer dereference when drmem is unavailable drm/etnaviv: rework perfmon query infrastructure rtc: omap: Use define directive for PIN_CONFIG_ACTIVE_HIGH selftests: vm: drop dependencies on page flags from mlock2 tests arm64: armv8_deprecated: Fix undef_hook mask for thumb setend scsi: zfcp: fix missing erp_lock in port recovery trigger for point-to-point dm verity fec: fix memory leak in verity_fec_dtr dm writecache: add cond_resched to avoid CPU hangs arm64: dts: allwinner: h6: Fix PMU compatible net: qualcomm: rmnet: Allow configuration updates to existing devices mm: Use fixed constant in page_frag_alloc instead of size + 1 tools: gpio: Fix out-of-tree build regression x86/speculation: Remove redundant arch_smt_update() invocation powerpc/pseries: Drop pointless static qualifier in vpa_debugfs_init() erofs: correct the remaining shrink objects crypto: mxs-dcp - fix scatterlist linearization for hash btrfs: fix missing semaphore unlock in btrfs_sync_file btrfs: fix missing file extent item for hole after ranged fsync btrfs: drop block from cache on error in relocation btrfs: set update the uuid generation as soon as possible Btrfs: fix crash during unmount due to race with delayed inode workers mtd: spinand: Do not erase the block before writing a bad block marker mtd: spinand: Stop using spinand->oobbuf for buffering bad block markers CIFS: Fix bug which the return value by asynchronous read is error KVM: VMX: fix crash cleanup when KVM wasn't used KVM: x86: Gracefully handle __vmalloc() failure during VM allocation KVM: VMX: Always VMCLEAR in-use VMCSes during crash with kexec support KVM: x86: Allocate new rmap and large page tracking when moving memslot KVM: s390: vsie: Fix delivery of addressing exceptions KVM: s390: vsie: Fix region 1 ASCE sanity shadow address checks KVM: nVMX: Properly handle userspace interrupt window request x86/entry/32: Add missing ASM_CLAC to general_protection entry signal: Extend exec_id to 64bits ath9k: Handle txpower changes even when TPC is disabled MIPS: OCTEON: irq: Fix potential NULL pointer dereference MIPS/tlbex: Fix LDDIR usage in setup_pw() for Loongson-3 pstore: pstore_ftrace_seq_next should increase position index irqchip/versatile-fpga: Apply clear-mask earlier KEYS: reaching the keys quotas correctly tpm: tpm2_bios_measurements_next should increase position index tpm: tpm1_bios_measurements_next should increase position index tpm: Don't make log failures fatal PCI: endpoint: Fix for concurrent memory allocation in OB address region PCI: Add boot interrupt quirk mechanism for Xeon chipsets PCI/ASPM: Clear the correct bits when enabling L1 substates PCI: pciehp: Fix indefinite wait on sysfs requests nvme: Treat discovery subsystems as unique subsystems nvme-fc: Revert "add module to ops template to allow module references" thermal: devfreq_cooling: inline all stubs for CONFIG_DEVFREQ_THERMAL=n acpi/x86: ignore unspecified bit positions in the ACPI global lock field media: ti-vpe: cal: fix disable_irqs to only the intended target ALSA: hda/realtek - Add quirk for MSI GL63 ALSA: hda/realtek - Remove now-unnecessary XPS 13 headphone noise fixups ALSA: hda/realtek - Set principled PC Beep configuration for ALC256 ALSA: doc: Document PC Beep Hidden Register on Realtek ALC256 ALSA: pcm: oss: Fix regression by buffer overflow fix ALSA: ice1724: Fix invalid access for enumerated ctl items ALSA: hda: Fix potential access overflow in beep helper ALSA: hda: Add driver blacklist ALSA: usb-audio: Add mixer workaround for TRX40 and co usb: gadget: composite: Inform controller driver of self-powered usb: gadget: f_fs: Fix use after free issue as part of queue failure ASoC: topology: use name_prefix for new kcontrol ASoC: dpcm: allow start or stop during pause for backend ASoC: dapm: connect virtual mux with default value ASoC: fix regwmask slub: improve bit diffusion for freelist ptr obfuscation uapi: rename ext2_swab() to swab() and share globally in swab.h IB/mlx5: Replace tunnel mpls capability bits for tunnel_offloads btrfs: track reloc roots based on their commit root bytenr btrfs: remove a BUG_ON() from merge_reloc_roots() btrfs: qgroup: ensure qgroup_rescan_running is only set when the worker is at least queued block, bfq: fix use-after-free in bfq_idle_slice_timer_body locking/lockdep: Avoid recursion in lockdep_count_{for,back}ward_deps() firmware: fix a double abort case with fw_load_sysfs_fallback md: check arrays is suspended in mddev_detach before call quiesce operations irqchip/gic-v4: Provide irq_retrigger to avoid circular locking dependency usb: dwc3: core: add support for disabling SS instances in park mode media: i2c: ov5695: Fix power on and off sequences block: Fix use-after-free issue accessing struct io_cq genirq/irqdomain: Check pointer in irq_domain_alloc_irqs_hierarchy() efi/x86: Ignore the memory attributes table on i386 x86/boot: Use unsigned comparison for addresses gfs2: Don't demote a glock until its revokes are written pstore/platform: fix potential mem leak if pstore_init_fs failed libata: Remove extra scsi_host_put() in ata_scsi_add_hosts() media: i2c: video-i2c: fix build errors due to 'imply hwmon' PCI/switchtec: Fix init_completion race condition with poll_wait() selftests/x86/ptrace_syscall_32: Fix no-vDSO segfault sched: Avoid scale real weight down to zero irqchip/versatile-fpga: Handle chained IRQs properly block: keep bdi->io_pages in sync with max_sectors_kb for stacked devices x86: Don't let pgprot_modify() change the page encryption bit xhci: bail out early if driver can't accress host in resume null_blk: fix spurious IO errors after failed past-wp access null_blk: Handle null_add_dev() failures properly null_blk: Fix the null_add_dev() error path firmware: arm_sdei: fix double-lock on hibernate with shared events media: venus: hfi_parser: Ignore HEVC encoding for V1 cpufreq: imx6q: Fixes unwanted cpu overclocking on i.MX6ULL i2c: st: fix missing struct parameter description qlcnic: Fix bad kzalloc null test cxgb4/ptp: pass the sign of offset delta in FW CMD hinic: fix wrong para of wait_for_completion_timeout hinic: fix a bug of waitting for IO stopped net: vxge: fix wrong __VA_ARGS__ usage bus: sunxi-rsb: Return correct data when mixing 16-bit and 8-bit reads ARM: dts: sun8i-a83t-tbs-a711: HM5065 doesn't like such a high voltage ANDROID: build.config.allmodconfig: Re-enable XFS_FS FROMGIT: of: property: Add device link support for extcon ANDROID: GKI: arm64: gki_defconfig: enable CONFIG_MM_EVENT_STAT ANDROID: GKI: add fields from per-process mm event tracking feature ANDROID: GKI: fix ABI diffs caused by ION heap and pool vmstat additions UPSTREAM: GKI: panic/reboot: allow specifying reboot_mode for panic only ANDROID: GKI: of: property: Add device link support for phys property ANDROID: GKI: usb: phy: Fix ABI diff for usb_otg_state ANDROID: GKI: usb: phy: Fix ABI diff due to usb_phy.drive_dp_pulse ANDROID: GKI: usb: phy: Fix ABI diff for usb_phy_type and usb_phy.reset ANDROID: gki_defconfig: enable CONFIG_GPIO_SYSFS ANDROID: GKI: qcom: Fix compile issue when setting msm_lmh_dcvs as a module ANDROID: GKI: drivers: cpu_cooling: allow platform freq mitigation ANDROID: GKI: ASoC: Add locking in DAPM widget power update ANDROID: GKI: ASoC: jack: Fix buttons enum value ANDROID: GKI: ALSA: jack: Add support to report second microphone ANDROID: GKI: ALSA: jack: Update supported jack switch types ANDROID: GKI: ALSA: jack: update jack types ANDROID: GKI: Export symbols arm_cpuidle_suspend, cpuidle_dev and cpuidle_register_governor ANDROID: GKI: usb: hcd: Add USB atomic notifier callback for HC died error ANDROID: media: increase video max frame number BACKPORT: nvmem: core: add NVMEM_SYSFS Kconfig UPSTREAM: nvmem: add support for cell info UPSTREAM: nvmem: remove the global cell list UPSTREAM: nvmem: use kref UPSTREAM: nvmem: use list_for_each_entry_safe in nvmem_device_remove_all_cells() UPSTREAM: nvmem: provide nvmem_dev_name() ANDROID: GKI: Bulk ABI update ANDROID: GKI: cpuhotplug: adding hotplug enums for vendor code ANDROID: Incremental fs: Fix create_file performance ANDROID: build.config.common: Add BUILDTOOLS_PREBUILT_BIN UPSTREAM: kheaders: include only headers into kheaders_data.tar.xz UPSTREAM: kheaders: remove meaningless -R option of 'ls' ANDROID: GKI: of: platform: initialize of_reserved_mem ANDROID: driver: gpu: drm: add notifier for panel related events ANDROID: include: drm: support unicasting mipi cmds to dsi ctrls ANDROID: include: drm: increase DRM max property count to 64 BACKPORT: drm: Add HDMI colorspace property ANDROID: drm: edid: add support for additional CEA extension blocks BACKPORT: drm: Parse HDR metadata info from EDID BACKPORT: drm: Add HDR source metadata property BACKPORT: drm/dp_mst: Parse FEC capability on MST ports ANDROID: GKI: ABI update for DRM changes ANDROID: ABI: add missing elf variables to representation ANDROID: GKI: power_supply: Add PROP_MOISTURE_DETECTION_ENABLED ANDROID: include: drm: add the definitions for DP Link Compliance tests ANDROID: drivers: gpu: drm: fix bugs encountered while fuzzing FROMLIST: power_supply: Add additional health properties to the header UPSTREAM: power: supply: core: Update sysfs-class-power ABI document UPSTREAM: Merge remote-tracking branch 'aosp/upstream-f2fs-stable-linux-4.19.y' into android-4.19 (v5.7-rc1) ANDROID: drivers: gpu: drm: add support for secure framebuffer ANDROID: include: uapi: drm: add additional QCOM modifiers ANDROID: drm: dsi: add two DSI mode flags for BLLP ANDROID: include: uapi: drm: add additional drm mode flags UPSTREAM: drm: plug memory leak on drm_setup() failure UPSTREAM: drm: factor out drm_close_helper() function ANDROID: GKI: Bulk ABI update BACKPORT: nl80211: Add per peer statistics to compute FCS error rate ANDROID: GKI: sound: usb: Add snd_usb_enable_audio_stream/find_snd_usb_substream ANDROID: GKI: add dma-buf includes ANDROID: GKI: sched: struct fields for Per-Sched-domain over utilization ANDROID: GKI: Add vendor fields to root_domain ANDROID: gki_defconfig: Enable CONFIG_IRQ_TIME_ACCOUNTING ANDROID: fix allmodconfig build to use the right toolchain ANDROID: fix allmodconfig build to use the right toolchain ANDROID: GKI: Update ABI Revert "UPSTREAM: mm, page_alloc: spread allocations across zones before introducing fragmentation" Revert "UPSTREAM: mm: use alloc_flags to record if kswapd can wake" Revert "BACKPORT: mm: move zone watermark accesses behind an accessor" Revert "BACKPORT: mm: reclaim small amounts of memory when an external fragmentation event occurs" Revert "BACKPORT: mm, compaction: be selective about what pageblocks to clear skip hints" ANDROID: GKI: panic: add vendor callback function in panic() UPSTREAM: GKI: thermal: make device_register's type argument const ANDROID: GKI: add base.h include to match MODULE_VERSIONS ANDROID: update the ABI based on the new whitelist ANDROID: GKI: fdt: export symbols required by modules ANDROID: GKI: drivers: of: Add APIs to find DDR device rank, HBB ANDROID: GKI: security: Add mmap export symbols for modules ANDROID: GKI: arch: add stub symbols for boot_reason and cold_boot ANDROID: GKI: USB: Fix ABI diff for struct usb_bus ANDROID: GKI: USB: Resolve ABI diff for usb_gadget and usb_gadget_ops ANDROID: GKI: add hidden V4L2_MEM2MEM_DEV ANDROID: GKI: enable VIDEO_V4L2_SUBDEV_API ANDROID: GKI: export symbols from abi_gki_aarch64_qcom_whitelist ANDROID: Update the whitelist for qcom SoCs ANDROID: Incremental fs: Fix compound page usercopy crash ANDROID: Incremental fs: Clean up incfs_test build process ANDROID: Incremental fs: make remount log buffer change atomic ANDROID: Incremental fs: Optimize get_filled_block ANDROID: Incremental fs: Fix mislabeled __user ptrs ANDROID: Incremental fs: Use 64-bit int for file_size when writing hash blocks Linux 4.19.115 drm/msm: Use the correct dma_sync calls in msm_gem drm_dp_mst_topology: fix broken drm_dp_sideband_parse_remote_dpcd_read() usb: dwc3: don't set gadget->is_otg flag rpmsg: glink: Remove chunk size word align warning arm64: Fix size of __early_cpu_boot_status drm/msm: stop abusing dma_map/unmap for cache clk: qcom: rcg: Return failure for RCG update fbcon: fix null-ptr-deref in fbcon_switch RDMA/cm: Update num_paths in cma_resolve_iboe_route error flow Bluetooth: RFCOMM: fix ODEBUG bug in rfcomm_dev_ioctl RDMA/cma: Teach lockdep about the order of rtnl and lock RDMA/ucma: Put a lock around every call to the rdma_cm layer ceph: canonicalize server path in place ceph: remove the extra slashes in the server path IB/hfi1: Fix memory leaks in sysfs registration and unregistration IB/hfi1: Call kobject_put() when kobject_init_and_add() fails ASoC: jz4740-i2s: Fix divider written at incorrect offset in register hwrng: imx-rngc - fix an error path tools/accounting/getdelays.c: fix netlink attribute length usb: dwc3: gadget: Wrap around when skip TRBs random: always use batched entropy for get_random_u{32,64} mlxsw: spectrum_flower: Do not stop at FLOW_ACTION_VLAN_MANGLE slcan: Don't transmit uninitialized stack data in padding net: stmmac: dwmac1000: fix out-of-bounds mac address reg setting net: phy: micrel: kszphy_resume(): add delay after genphy_resume() before accessing PHY registers net: dsa: bcm_sf2: Ensure correct sub-node is parsed net: dsa: bcm_sf2: Do not register slave MDIO bus with OF ipv6: don't auto-add link-local address to lag ports mm: mempolicy: require at least one nodeid for MPOL_PREFERRED include/linux/notifier.h: SRCU: fix ctags bitops: protect variables in set_mask_bits() macro padata: always acquire cpu_hotplug_lock before pinst->lock net: Fix Tx hash bound checking rxrpc: Fix sendmsg(MSG_WAITALL) handling ALSA: hda/ca0132 - Add Recon3Di quirk to handle integrated sound on EVGA X99 Classified motherboard power: supply: axp288_charger: Add special handling for HP Pavilion x2 10 extcon: axp288: Add wakeup support mei: me: add cedar fork device ids coresight: do not use the BIT() macro in the UAPI header misc: pci_endpoint_test: Avoid using module parameter to determine irqtype misc: pci_endpoint_test: Fix to support > 10 pci-endpoint-test devices misc: rtsx: set correct pcr_ops for rts522A media: rc: IR signal for Panasonic air conditioner too long drm/etnaviv: replace MMU flush marker with flush sequence tools/power turbostat: Fix missing SYS_LPI counter on some Chromebooks tools/power turbostat: Fix gcc build warnings drm/amdgpu: fix typo for vcn1 idle check initramfs: restore default compression behavior drm/bochs: downgrade pci_request_region failure from error to warning drm/amd/display: Add link_rate quirk for Apple 15" MBP 2017 nvme-rdma: Avoid double freeing of async event data sctp: fix possibly using a bad saddr with a given dst sctp: fix refcount bug in sctp_wfree net, ip_tunnel: fix interface lookup with no key ipv4: fix a RCU-list lock in fib_triestat_seq_show ANDROID: GKI: export symbols required by SPECTRA_CAMERA ANDROID: GKI: ARM/ARM64: Introduce arch_read_hardware_id ANDROID: GKI: drivers: base: soc: export symbols for socinfo ANDROID: GKI: Update ABI ANDROID: GKI: ASoC: msm: fix integer overflow for long duration offload playback ANDROID: GKI: Bulk ABI update Revert "ANDROID: GKI: mm: add struct/enum fields for SPECULATIVE_PAGE_FAULTS" ANDROID: GKI: Revert "arm64: kill flush_cache_all()" ANDROID: GKI: Revert "arm64: Remove unused macros from assembler.h" ANDROID: GKI: kernel/dma, mm/cma: Export symbols needed by vendor modules ANDROID: GKI: mm: Export symbols __next_zones_zonelist and zone_watermark_ok_safe ANDROID: GKI: mm/memblock: export memblock_overlaps_memory ANDROID: GKI: net, skbuff: export symbols needed by vendor drivers ANDROID: GKI: Add stub __cpu_isolated_mask symbol ANDROID: GKI: sched: stub sched_isolate symbols ANDROID: GKI: export saved_command_line ANDROID: GKI: Update ABI ANDROID: GKI: ASoC: core: Update ALSA core to issue restart in underrun. ANDROID: GKI: SoC: pcm: Add a restart callback field to struct snd_pcm_ops ANDROID: GKI: SoC: pcm: Add fields to struct snd_pcm_ops and struct snd_soc_component_driver ANDROID: GKI: ASoC: core: Add compat_ioctl callback to struct snd_pcm_ops ANDROID: GKI: ALSA: core: modify, rename and export create_subdir API ANDROID: GKI: usb: Add helper API to issue stop endpoint command ANDROID: GKI: Thermal: thermal_zone_get_cdev_by_name added ANDROID: GKI: add missing exports for CONFIG_ARM_SMMU=m ANDROID: power: wakeup_reason: wake reason enhancements BACKPORT: FROMGIT: kbuild: mkcompile_h: Include $LD version in /proc/version ANDROID: GKI: kernel: Export symbols needed by msm_minidump.ko and minidump_log.ko ubifs: wire up FS_IOC_GET_ENCRYPTION_NONCE f2fs: wire up FS_IOC_GET_ENCRYPTION_NONCE ext4: wire up FS_IOC_GET_ENCRYPTION_NONCE fscrypt: add FS_IOC_GET_ENCRYPTION_NONCE ioctl ANDROID: Bulk update the ABI xml ANDROID: gki_defconfig: add CONFIG_IPV6_SUBTREES ANDROID: GKI: arm64: reserve space in cpu_hwcaps and cpu_hwcap_keys arrays ANDROID: GKI: of: reserved_mem: Fix kmemleak crash on no-map region ANDROID: GKI: sched: add task boost vendor fields to task_struct ANDROID: GKI: mm: add rss counter for unreclaimable pages ANDROID: GKI: irqdomain: add bus token DOMAIN_BUS_WAKEUP ANDROID: GKI: arm64: fault: do_tlb_conf_fault_cb register fault callback ANDROID: GKI: QoS: Enhance framework to support cpu/irq specific QoS requests ANDROID: GKI: Bulk ABI update ANDROID: GKI: PM/devfreq: Do not switch governors from sysfs when device is suspended ANDROID: GKI: PM / devfreq: Fix race condition between suspend/resume and governor_store ANDROID: GKI: PM / devfreq: Introduce a sysfs lock ANDROID: GKI: regmap: irq: Add support to clear ack registers ANDROID: GKI: Remove SCHED_AUTOGROUP ANDROID: ignore compiler tag __must_check for GENKSYMS ANDROID: GKI: Bulk update ABI ANDROID: GKI: Fix ABI diff for struct thermal_cooling_device_ops ANDROID: GKI: ASoC: soc-core: export function to find components ANDROID: GKI: thermal: thermal_sys: Add configurable thermal trip points. ANDROID: fscrypt: fall back to filesystem-layer crypto when needed ANDROID: block: require drivers to declare supported crypto key type(s) ANDROID: block: make blk_crypto_start_using_mode() properly check for support ANDROID: GKI: power: supply: format regression ANDROID: GKI: kobject: increase number of kobject uevent pointers to 64 ANDROID: GKI: drivers: video: backlight: Fix ABI diff for struct backlight_device ANDROID: GKI: usb: xhci: Add support for secondary interrupters ANDROID: GKI: usb: host: xhci: Add support for usb core indexing ANDROID: gki_defconfig: enable USB_XHCI_HCD ANDROID: gki_defconfig: enable CONFIG_BRIDGE ANDROID: GKI: Update ABI report ANDROID: GKI: arm64: smp: Add set_update_ipi_history_callback ANDROID: kbuild: ensure __cfi_check is correctly aligned f2fs: keep inline_data when compression conversion f2fs: fix to disable compression on directory f2fs: add missing CONFIG_F2FS_FS_COMPRESSION f2fs: switch discard_policy.timeout to bool type f2fs: fix to verify tpage before releasing in f2fs_free_dic() f2fs: show compression in statx f2fs: clean up dic->tpages assignment f2fs: compress: support zstd compress algorithm f2fs: compress: add .{init,destroy}_decompress_ctx callback f2fs: compress: fix to call missing destroy_compress_ctx() f2fs: change default compression algorithm f2fs: clean up {cic,dic}.ref handling f2fs: fix to use f2fs_readpage_limit() in f2fs_read_multi_pages() f2fs: xattr.h: Make stub helpers inline f2fs: fix to avoid double unlock f2fs: fix potential .flags overflow on 32bit architecture f2fs: fix NULL pointer dereference in f2fs_verity_work() f2fs: fix to clear PG_error if fsverity failed f2fs: don't call fscrypt_get_encryption_info() explicitly in f2fs_tmpfile() f2fs: don't trigger data flush in foreground operation f2fs: fix NULL pointer dereference in f2fs_write_begin() f2fs: clean up f2fs_may_encrypt() f2fs: fix to avoid potential deadlock f2fs: don't change inode status under page lock f2fs: fix potential deadlock on compressed quota file f2fs: delete DIO read lock f2fs: don't mark compressed inode dirty during f2fs_iget() f2fs: fix to account compressed blocks in f2fs_compressed_blocks() f2fs: xattr.h: Replace zero-length array with flexible-array member f2fs: fix to update f2fs_super_block fields under sb_lock f2fs: Add a new CP flag to help fsck fix resize SPO issues f2fs: Fix mount failure due to SPO after a successful online resize FS f2fs: use kmem_cache pool during inline xattr lookups f2fs: skip migration only when BG_GC is called f2fs: fix to show tracepoint correctly f2fs: avoid __GFP_NOFAIL in f2fs_bio_alloc f2fs: introduce F2FS_IOC_GET_COMPRESS_BLOCKS f2fs: fix to avoid triggering IO in write path f2fs: add prefix for f2fs slab cache name f2fs: introduce DEFAULT_IO_TIMEOUT f2fs: skip GC when section is full f2fs: add migration count iff migration happens f2fs: clean up bggc mount option f2fs: clean up lfs/adaptive mount option f2fs: fix to show norecovery mount option f2fs: clean up parameter of macro XATTR_SIZE() f2fs: clean up codes with {f2fs_,}data_blkaddr() f2fs: show mounted time f2fs: Use scnprintf() for avoiding potential buffer overflow f2fs: allow to clear F2FS_COMPR_FL flag f2fs: fix to check dirty pages during compressed inode conversion f2fs: fix to account compressed inode correctly f2fs: fix wrong check on F2FS_IOC_FSSETXATTR f2fs: fix to avoid use-after-free in f2fs_write_multi_pages() f2fs: fix to avoid using uninitialized variable f2fs: fix inconsistent comments f2fs: remove i_sem lock coverage in f2fs_setxattr() f2fs: cover last_disk_size update with spinlock f2fs: fix to check i_compr_blocks correctly FROMLIST: kmod: make request_module() return an error when autoloading is disabled ANDROID: GKI: Update ABI report ANDROID: GKI: ARM64: dma-mapping: export symbol arch_setup_dma_ops ANDROID: GKI: ARM: dma-mapping: export symbol arch_setup_dma_ops ANDROID: GKI: ASoC: dapm: Avoid static route b/w cpu and codec dai ANDROID: GKI: ASoC: pcm: Add support for hostless playback/capture ANDROID: GKI: ASoC: core - add hostless DAI support ANDROID: GKI: drivers: thermal: Resolve ABI diff for struct thermal_zone_device_ops ANDROID: GKI: drivers: thermal: Add support for getting trip temperature ANDROID: GKI: Add functions of_thermal_handle_trip/of_thermal_handle_trip_temp ANDROID: GKI: drivers: thermal: Add post suspend evaluate flag to thermal zone devicetree UPSTREAM: loop: Only freeze block queue when needed. UPSTREAM: loop: Only change blocksize when needed. ANDROID: Fix wq fp check for CFI builds ANDROID: GKI: update abi definition after CONFIG_DEBUG_LIST was enabled ANDROID: gki_defconfig: enable CONFIG_DEBUG_LIST ANDROID: GKI: Update ABI definition ANDROID: GKI: remove condition causing sk_buff struct ABI differences ANDROID: GKI: Export symbol arch_timer_mem_get_cval ANDROID: GKI: pwm: core: Add option to config PWM duty/period with u64 data length ANDROID: Update ABI whitelist for qcom SoCs ANDROID: Incremental fs: Fix remount ANDROID: Incremental fs: Protect get_fill_block, and add a field ANDROID: Incremental fs: Fix crash polling 0 size read_log ANDROID: Incremental fs: get_filled_blocks: better index_out ANDROID: GKI: of: property: Add device links support for "qcom,wrapper-dev" ANDROID: GKI: update abi definitions due to recent changes ANDROID: GKI: clk: Initialize in stack clk_init_data to 0 in all drivers ANDROID: GKI: drivers: clksource: Add API to return cval ANDROID: GKI: clk: Add support for voltage voting ANDROID: GKI: kernel: Export task and IRQ affinity symbols ANDROID: GKI: regulator: core: Add support for regulator providers with sync state ANDROID: GKI: regulator: Call proxy-consumer functions for each regulator registered ANDROID: GKI: regulator: Add proxy consumer driver ANDROID: GKI: regulator: core: allow long device tree supply regulator property names ANDROID: GKI: Revert "regulator: Enable supply regulator if child rail is enabled." ANDROID: GKI: regulator: Remove redundant set_mode call in drms_uA_update ANDROID: GKI: net: Add the get current NAPI context API ANDROID: GKI: remove DRM_KMS_CMA_HELPER from GKI configuration ANDROID: GKI: edac: Fix ABI diffs in edac_device_ctl_info struct ANDROID: GKI: pwm: Add different PWM output types support UPSTREAM: cfg80211: Authentication offload to user space in AP mode Linux 4.19.114 arm64: dts: ls1046ardb: set RGMII interfaces to RGMII_ID mode arm64: dts: ls1043a-rdb: correct RGMII delay mode to rgmii-id ARM: dts: N900: fix onenand timings ARM: dts: imx6: phycore-som: fix arm and soc minimum voltage ARM: bcm2835-rpi-zero-w: Add missing pinctrl name ARM: dts: oxnas: Fix clear-mask property perf map: Fix off by one in strncpy() size argument arm64: alternative: fix build with clang integrated assembler net: ks8851-ml: Fix IO operations, again gpiolib: acpi: Add quirk to ignore EC wakeups on HP x2 10 CHT + AXP288 model bpf: Explicitly memset some bpf info structures declared on the stack bpf: Explicitly memset the bpf_attr structure platform/x86: pmc_atom: Add Lex 2I385SW to critclk_systems DMI table vt: vt_ioctl: fix use-after-free in vt_in_use() vt: vt_ioctl: fix VT_DISALLOCATE freeing in-use virtual console vt: vt_ioctl: remove unnecessary console allocation checks vt: switch vt_dont_switch to bool vt: ioctl, switch VT_IS_IN_USE and VT_BUSY to inlines vt: selection, introduce vc_is_sel mac80211: fix authentication with iwlwifi/mvm mac80211: Check port authorization in the ieee80211_tx_dequeue() case media: xirlink_cit: add missing descriptor sanity checks media: stv06xx: add missing descriptor sanity checks media: dib0700: fix rc endpoint lookup media: ov519: add missing endpoint sanity checks libfs: fix infoleak in simple_attr_read() ahci: Add Intel Comet Lake H RAID PCI ID staging: wlan-ng: fix use-after-free Read in hfa384x_usbin_callback staging: wlan-ng: fix ODEBUG bug in prism2sta_disconnect_usb staging: rtl8188eu: Add ASUS USB-N10 Nano B1 to device table media: usbtv: fix control-message timeouts media: flexcop-usb: fix endpoint sanity check usb: musb: fix crash with highmen PIO and usbmon USB: serial: io_edgeport: fix slab-out-of-bounds read in edge_interrupt_callback USB: cdc-acm: restore capability check order USB: serial: option: add Wistron Neweb D19Q1 USB: serial: option: add BroadMobi BM806U USB: serial: option: add support for ASKEY WWHC050 mac80211: set IEEE80211_TX_CTRL_PORT_CTRL_PROTO for nl80211 TX mac80211: add option for setting control flags Revert "r8169: check that Realtek PHY driver module is loaded" vti6: Fix memory leak of skb if input policy check fails bpf/btf: Fix BTF verification of enum members in struct/union netfilter: nft_fwd_netdev: validate family and chain type netfilter: flowtable: reload ip{v6}h in nf_flow_tuple_ip{v6} afs: Fix some tracing details xfrm: policy: Fix doulbe free in xfrm_policy_timer xfrm: add the missing verify_sec_ctx_len check in xfrm_add_acquire xfrm: fix uctx len check in verify_sec_ctx_len RDMA/mlx5: Block delay drop to unprivileged users vti[6]: fix packet tx through bpf_redirect() in XinY cases xfrm: handle NETDEV_UNREGISTER for xfrm device genirq: Fix reference leaks on irq affinity notifiers RDMA/core: Ensure security pkey modify is not lost gpiolib: acpi: Add quirk to ignore EC wakeups on HP x2 10 BYT + AXP288 model gpiolib: acpi: Rework honor_wakeup option into an ignore_wake option gpiolib: acpi: Correct comment for HP x2 10 honor_wakeup quirk mac80211: mark station unauthorized before key removal nl80211: fix NL80211_ATTR_CHANNEL_WIDTH attribute type scsi: sd: Fix optimal I/O size for devices that change reported values scripts/dtc: Remove redundant YYLOC global declaration tools: Let O= makes handle a relative path with -C option perf probe: Do not depend on dwfl_module_addrsym() ARM: dts: omap5: Add bus_dma_limit for L3 bus ARM: dts: dra7: Add bus_dma_limit for L3 bus ceph: check POOL_FLAG_FULL/NEARFULL in addition to OSDMAP_FULL/NEARFULL Input: avoid BIT() macro usage in the serio.h UAPI header Input: synaptics - enable RMI on HP Envy 13-ad105ng Input: raydium_i2c_ts - fix error codes in raydium_i2c_boot_trigger() i2c: hix5hd2: add missed clk_disable_unprepare in remove ftrace/x86: Anotate text_mutex split between ftrace_arch_code_modify_post_process() and ftrace_arch_code_modify_prepare() sxgbe: Fix off by one in samsung driver strncpy size arg dpaa_eth: Remove unnecessary boolean expression in dpaa_get_headroom mac80211: Do not send mesh HWMP PREQ if HWMP is disabled scsi: ipr: Fix softlockup when rescanning devices in petitboot s390/qeth: handle error when backing RX buffer fsl/fman: detect FMan erratum A050385 arm64: dts: ls1043a: FMan erratum A050385 dt-bindings: net: FMan erratum A050385 cgroup1: don't call release_agent when it is "" drivers/of/of_mdio.c:fix of_mdiobus_register() cpupower: avoid multiple definition with gcc -fno-common nfs: add minor version to nfs_server_key for fscache cgroup-v1: cgroup_pidlist_next should update position index hsr: set .netnsok flag hsr: add restart routine into hsr_get_node_list() hsr: use rcu_read_lock() in hsr_get_node_{list/status}() vxlan: check return value of gro_cells_init() tcp: repair: fix TCP_QUEUE_SEQ implementation r8169: re-enable MSI on RTL8168c net: phy: mdio-mux-bcm-iproc: check clk_prepare_enable() return value net: dsa: mt7530: Change the LINK bit to reflect the link status net: ip_gre: Accept IFLA_INFO_DATA-less configuration net: ip_gre: Separate ERSPAN newlink / changelink callbacks bnxt_en: Reset rings if ring reservation fails during open() bnxt_en: fix memory leaks in bnxt_dcbnl_ieee_getets() slcan: not call free_netdev before rtnl_unlock in slcan_open NFC: fdp: Fix a signedness bug in fdp_nci_send_patch() net: stmmac: dwmac-rk: fix error path in rk_gmac_probe net_sched: keep alloc_hash updated after hash allocation net_sched: cls_route: remove the right filter from hashtable net: qmi_wwan: add support for ASKEY WWHC050 net/packet: tpacket_rcv: avoid a producer race condition net: mvneta: Fix the case where the last poll did not process all rx net: dsa: Fix duplicate frames flooded by learning net: cbs: Fix software cbs to consider packet sending time mlxsw: spectrum_mr: Fix list iteration in error path macsec: restrict to ethernet devices hsr: fix general protection fault in hsr_addr_is_self() geneve: move debug check after netdev unregister Revert "drm/dp_mst: Skip validating ports during destruction, just ref" mmc: sdhci-tegra: Fix busy detection by enabling MMC_CAP_NEED_RSP_BUSY mmc: sdhci-omap: Fix busy detection by enabling MMC_CAP_NEED_RSP_BUSY mmc: core: Respect MMC_CAP_NEED_RSP_BUSY for eMMC sleep command mmc: core: Respect MMC_CAP_NEED_RSP_BUSY for erase/trim/discard mmc: core: Allow host controllers to require R1B for CMD6 f2fs: fix to avoid potential deadlock f2fs: add missing function name in kernel message f2fs: recycle unused compress_data.chksum feild f2fs: fix to avoid NULL pointer dereference f2fs: fix leaking uninitialized memory in compressed clusters f2fs: fix the panic in do_checkpoint() f2fs: fix to wait all node page writeback mm/swapfile.c: move inode_lock out of claim_swapfile fscrypt: don't evict dirty inodes after removing key Conflicts: Documentation/arm64/silicon-errata.txt Documentation/devicetree/bindings Documentation/devicetree/bindings/net/fsl-fman.txt arch/arm/kernel/setup.c arch/arm/kernel/smp.c arch/arm/mm/dma-mapping.c arch/arm64/Kconfig arch/arm64/Makefile arch/arm64/include/asm/cpucaps.h arch/arm64/include/asm/cputype.h arch/arm64/include/asm/proc-fns.h arch/arm64/include/asm/traps.h arch/arm64/kernel/arm64ksyms.c arch/arm64/kernel/cpu_errata.c arch/arm64/kernel/setup.c arch/arm64/kernel/smp.c arch/arm64/mm/dma-mapping.c arch/arm64/mm/fault.c arch/arm64/mm/proc.S drivers/base/power/wakeup.c drivers/clk/clk.c drivers/clk/qcom/clk-rcg2.c drivers/clocksource/arm_arch_timer.c drivers/devfreq/devfreq.c drivers/devfreq/governor_simpleondemand.c drivers/dma-buf/dma-buf.c drivers/extcon/extcon.c drivers/gpu/Makefile drivers/gpu/drm/drm_connector.c drivers/gpu/drm/drm_dp_mst_topology.c drivers/gpu/drm/drm_edid.c drivers/gpu/drm/drm_file.c drivers/gpu/drm/drm_panel.c drivers/gpu/drm/drm_property.c drivers/iommu/Kconfig drivers/iommu/Makefile drivers/iommu/arm-smmu.c drivers/iommu/dma-iommu.c drivers/iommu/dma-mapping-fast.c drivers/iommu/io-pgtable-arm.c drivers/iommu/io-pgtable-fast.c drivers/iommu/io-pgtable.c drivers/iommu/iommu.c drivers/irqchip/irq-gic-v3.c drivers/media/v4l2-core/v4l2-ioctl.c drivers/mmc/core/Kconfig drivers/mmc/core/block.c drivers/mmc/core/queue.c drivers/mmc/host/cqhci.c drivers/mmc/host/sdhci-msm.c drivers/net/wireless/ath/wil6210/interrupt.c drivers/net/wireless/ath/wil6210/main.c drivers/net/wireless/ath/wil6210/wil6210.h drivers/net/wireless/ath/wil6210/wmi.c drivers/nvmem/core.c drivers/nvmem/nvmem-sysfs.c drivers/of/fdt.c drivers/power/supply/power_supply_sysfs.c drivers/pwm/sysfs.c drivers/regulator/core.c drivers/scsi/sd.c drivers/scsi/ufs/ufshcd.c drivers/tty/serial/Kconfig drivers/tty/serial/Makefile drivers/usb/common/common.c fs/crypto/crypto.c fs/f2fs/checkpoint.c fs/f2fs/f2fs.h include/drm/drm_connector.h include/drm/drm_dp_mst_helper.h include/drm/drm_panel.h include/linux/clk-provider.h include/linux/dma-buf.h include/linux/dma-mapping-fast.h include/linux/dma-mapping.h include/linux/extcon.h include/linux/io-pgtable.h include/linux/iommu.h include/linux/kobject.h include/linux/mm.h include/linux/mm_types.h include/linux/mmc/host.h include/linux/netdevice.h include/linux/power_supply.h include/linux/pwm.h include/linux/regulator/driver.h include/linux/thermal.h include/linux/vm_event_item.h include/net/cfg80211.h include/scsi/scsi_device.h include/sound/pcm.h include/sound/soc.h include/uapi/drm/drm_mode.h include/uapi/linux/coresight-stm.h include/uapi/linux/ip.h include/uapi/linux/nl80211.h include/uapi/linux/videodev2.h include/uapi/sound/compress_offload.h kernel/dma/coherent.c kernel/dma/mapping.c kernel/panic.c kernel/power/qos.c kernel/sched/sched.h mm/Kconfig mm/filemap.c mm/swapfile.c mm/vmalloc.c mm/vmstat.c net/qrtr/qrtr.c net/wireless/nl80211.c net/wireless/scan.c sound/core/compress_offload.c sound/soc/soc-core.c sound/usb/card.c sound/usb/pcm.c sound/usb/pcm.h sound/usb/usbaudio.h Fixed build errors: drivers/base/power/main.c drivers/thermal/thermal_core.c drivers/cpuidle/lpm-levels.c include/soc/qcom/lpm_levels.h Change-Id: Idf25b239f53681bdfa2ef371a91720fadf1a3f01 Signed-off-by: Srinivasarao P <spathi@codeaurora.org>
3518 lines
87 KiB
C
3518 lines
87 KiB
C
/*
|
|
* linux/mm/vmalloc.c
|
|
*
|
|
* Copyright (C) 1993 Linus Torvalds
|
|
* Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
|
|
* SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
|
|
* Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
|
|
* Numa awareness, Christoph Lameter, SGI, June 2005
|
|
*/
|
|
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/module.h>
|
|
#include <linux/highmem.h>
|
|
#include <linux/sched/signal.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/proc_fs.h>
|
|
#include <linux/seq_file.h>
|
|
#include <linux/debugobjects.h>
|
|
#include <linux/kallsyms.h>
|
|
#include <linux/list.h>
|
|
#include <linux/notifier.h>
|
|
#include <linux/rbtree.h>
|
|
#include <linux/radix-tree.h>
|
|
#include <linux/rcupdate.h>
|
|
#include <linux/pfn.h>
|
|
#include <linux/kmemleak.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/compiler.h>
|
|
#include <linux/llist.h>
|
|
#include <linux/bitops.h>
|
|
#include <linux/rbtree_augmented.h>
|
|
#include <linux/overflow.h>
|
|
|
|
#include <linux/uaccess.h>
|
|
#include <asm/tlbflush.h>
|
|
#include <asm/shmparam.h>
|
|
|
|
#include "internal.h"
|
|
|
|
struct vfree_deferred {
|
|
struct llist_head list;
|
|
struct work_struct wq;
|
|
};
|
|
static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
|
|
|
|
static void __vunmap(const void *, int);
|
|
|
|
static void free_work(struct work_struct *w)
|
|
{
|
|
struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
|
|
struct llist_node *t, *llnode;
|
|
|
|
llist_for_each_safe(llnode, t, llist_del_all(&p->list))
|
|
__vunmap((void *)llnode, 1);
|
|
}
|
|
|
|
/*** Page table manipulation functions ***/
|
|
|
|
static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
|
|
{
|
|
pte_t *pte;
|
|
|
|
pte = pte_offset_kernel(pmd, addr);
|
|
do {
|
|
pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
|
|
WARN_ON(!pte_none(ptent) && !pte_present(ptent));
|
|
} while (pte++, addr += PAGE_SIZE, addr != end);
|
|
}
|
|
|
|
static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
|
|
{
|
|
pmd_t *pmd;
|
|
unsigned long next;
|
|
|
|
pmd = pmd_offset(pud, addr);
|
|
do {
|
|
next = pmd_addr_end(addr, end);
|
|
if (pmd_clear_huge(pmd))
|
|
continue;
|
|
if (pmd_none_or_clear_bad(pmd))
|
|
continue;
|
|
vunmap_pte_range(pmd, addr, next);
|
|
} while (pmd++, addr = next, addr != end);
|
|
}
|
|
|
|
static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end)
|
|
{
|
|
pud_t *pud;
|
|
unsigned long next;
|
|
|
|
pud = pud_offset(p4d, addr);
|
|
do {
|
|
next = pud_addr_end(addr, end);
|
|
if (pud_clear_huge(pud))
|
|
continue;
|
|
if (pud_none_or_clear_bad(pud))
|
|
continue;
|
|
vunmap_pmd_range(pud, addr, next);
|
|
} while (pud++, addr = next, addr != end);
|
|
}
|
|
|
|
static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end)
|
|
{
|
|
p4d_t *p4d;
|
|
unsigned long next;
|
|
|
|
p4d = p4d_offset(pgd, addr);
|
|
do {
|
|
next = p4d_addr_end(addr, end);
|
|
if (p4d_clear_huge(p4d))
|
|
continue;
|
|
if (p4d_none_or_clear_bad(p4d))
|
|
continue;
|
|
vunmap_pud_range(p4d, addr, next);
|
|
} while (p4d++, addr = next, addr != end);
|
|
}
|
|
|
|
static void vunmap_page_range(unsigned long addr, unsigned long end)
|
|
{
|
|
pgd_t *pgd;
|
|
unsigned long next;
|
|
|
|
BUG_ON(addr >= end);
|
|
pgd = pgd_offset_k(addr);
|
|
do {
|
|
next = pgd_addr_end(addr, end);
|
|
if (pgd_none_or_clear_bad(pgd))
|
|
continue;
|
|
vunmap_p4d_range(pgd, addr, next);
|
|
} while (pgd++, addr = next, addr != end);
|
|
}
|
|
|
|
static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
|
|
unsigned long end, pgprot_t prot, struct page **pages, int *nr)
|
|
{
|
|
pte_t *pte;
|
|
|
|
/*
|
|
* nr is a running index into the array which helps higher level
|
|
* callers keep track of where we're up to.
|
|
*/
|
|
|
|
pte = pte_alloc_kernel(pmd, addr);
|
|
if (!pte)
|
|
return -ENOMEM;
|
|
do {
|
|
struct page *page = pages[*nr];
|
|
|
|
if (WARN_ON(!pte_none(*pte)))
|
|
return -EBUSY;
|
|
if (WARN_ON(!page))
|
|
return -ENOMEM;
|
|
set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
|
|
(*nr)++;
|
|
} while (pte++, addr += PAGE_SIZE, addr != end);
|
|
return 0;
|
|
}
|
|
|
|
static int vmap_pmd_range(pud_t *pud, unsigned long addr,
|
|
unsigned long end, pgprot_t prot, struct page **pages, int *nr)
|
|
{
|
|
pmd_t *pmd;
|
|
unsigned long next;
|
|
|
|
pmd = pmd_alloc(&init_mm, pud, addr);
|
|
if (!pmd)
|
|
return -ENOMEM;
|
|
do {
|
|
next = pmd_addr_end(addr, end);
|
|
if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
|
|
return -ENOMEM;
|
|
} while (pmd++, addr = next, addr != end);
|
|
return 0;
|
|
}
|
|
|
|
static int vmap_pud_range(p4d_t *p4d, unsigned long addr,
|
|
unsigned long end, pgprot_t prot, struct page **pages, int *nr)
|
|
{
|
|
pud_t *pud;
|
|
unsigned long next;
|
|
|
|
pud = pud_alloc(&init_mm, p4d, addr);
|
|
if (!pud)
|
|
return -ENOMEM;
|
|
do {
|
|
next = pud_addr_end(addr, end);
|
|
if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
|
|
return -ENOMEM;
|
|
} while (pud++, addr = next, addr != end);
|
|
return 0;
|
|
}
|
|
|
|
static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
|
|
unsigned long end, pgprot_t prot, struct page **pages, int *nr)
|
|
{
|
|
p4d_t *p4d;
|
|
unsigned long next;
|
|
|
|
p4d = p4d_alloc(&init_mm, pgd, addr);
|
|
if (!p4d)
|
|
return -ENOMEM;
|
|
do {
|
|
next = p4d_addr_end(addr, end);
|
|
if (vmap_pud_range(p4d, addr, next, prot, pages, nr))
|
|
return -ENOMEM;
|
|
} while (p4d++, addr = next, addr != end);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
|
|
* will have pfns corresponding to the "pages" array.
|
|
*
|
|
* Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
|
|
*/
|
|
static int vmap_page_range_noflush(unsigned long start, unsigned long end,
|
|
pgprot_t prot, struct page **pages)
|
|
{
|
|
pgd_t *pgd;
|
|
unsigned long next;
|
|
unsigned long addr = start;
|
|
int err = 0;
|
|
int nr = 0;
|
|
|
|
BUG_ON(addr >= end);
|
|
pgd = pgd_offset_k(addr);
|
|
do {
|
|
next = pgd_addr_end(addr, end);
|
|
err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr);
|
|
if (err)
|
|
return err;
|
|
} while (pgd++, addr = next, addr != end);
|
|
|
|
return nr;
|
|
}
|
|
|
|
static int vmap_page_range(unsigned long start, unsigned long end,
|
|
pgprot_t prot, struct page **pages)
|
|
{
|
|
int ret;
|
|
|
|
ret = vmap_page_range_noflush(start, end, prot, pages);
|
|
flush_cache_vmap(start, end);
|
|
return ret;
|
|
}
|
|
|
|
int is_vmalloc_or_module_addr(const void *x)
|
|
{
|
|
/*
|
|
* ARM, x86-64 and sparc64 put modules in a special place,
|
|
* and fall back on vmalloc() if that fails. Others
|
|
* just put it in the vmalloc space.
|
|
*/
|
|
#if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
|
|
unsigned long addr = (unsigned long)x;
|
|
if (addr >= MODULES_VADDR && addr < MODULES_END)
|
|
return 1;
|
|
#endif
|
|
return is_vmalloc_addr(x);
|
|
}
|
|
|
|
/*
|
|
* Walk a vmap address to the struct page it maps.
|
|
*/
|
|
struct page *vmalloc_to_page(const void *vmalloc_addr)
|
|
{
|
|
unsigned long addr = (unsigned long) vmalloc_addr;
|
|
struct page *page = NULL;
|
|
pgd_t *pgd = pgd_offset_k(addr);
|
|
p4d_t *p4d;
|
|
pud_t *pud;
|
|
pmd_t *pmd;
|
|
pte_t *ptep, pte;
|
|
|
|
/*
|
|
* XXX we might need to change this if we add VIRTUAL_BUG_ON for
|
|
* architectures that do not vmalloc module space
|
|
*/
|
|
VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
|
|
|
|
if (pgd_none(*pgd))
|
|
return NULL;
|
|
p4d = p4d_offset(pgd, addr);
|
|
if (p4d_none(*p4d))
|
|
return NULL;
|
|
pud = pud_offset(p4d, addr);
|
|
|
|
/*
|
|
* Don't dereference bad PUD or PMD (below) entries. This will also
|
|
* identify huge mappings, which we may encounter on architectures
|
|
* that define CONFIG_HAVE_ARCH_HUGE_VMAP=y. Such regions will be
|
|
* identified as vmalloc addresses by is_vmalloc_addr(), but are
|
|
* not [unambiguously] associated with a struct page, so there is
|
|
* no correct value to return for them.
|
|
*/
|
|
WARN_ON_ONCE(pud_bad(*pud));
|
|
if (pud_none(*pud) || pud_bad(*pud))
|
|
return NULL;
|
|
pmd = pmd_offset(pud, addr);
|
|
WARN_ON_ONCE(pmd_bad(*pmd));
|
|
if (pmd_none(*pmd) || pmd_bad(*pmd))
|
|
return NULL;
|
|
|
|
ptep = pte_offset_map(pmd, addr);
|
|
pte = *ptep;
|
|
if (pte_present(pte))
|
|
page = pte_page(pte);
|
|
pte_unmap(ptep);
|
|
return page;
|
|
}
|
|
EXPORT_SYMBOL(vmalloc_to_page);
|
|
|
|
/*
|
|
* Map a vmalloc()-space virtual address to the physical page frame number.
|
|
*/
|
|
unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
|
|
{
|
|
return page_to_pfn(vmalloc_to_page(vmalloc_addr));
|
|
}
|
|
EXPORT_SYMBOL(vmalloc_to_pfn);
|
|
|
|
|
|
/*** Global kva allocator ***/
|
|
|
|
#define DEBUG_AUGMENT_PROPAGATE_CHECK 0
|
|
#define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0
|
|
|
|
#define VM_LAZY_FREE 0x02
|
|
#define VM_VM_AREA 0x04
|
|
|
|
static DEFINE_SPINLOCK(vmap_area_lock);
|
|
/* Export for kexec only */
|
|
LIST_HEAD(vmap_area_list);
|
|
static LLIST_HEAD(vmap_purge_list);
|
|
static struct rb_root vmap_area_root = RB_ROOT;
|
|
static bool vmap_initialized __read_mostly;
|
|
|
|
/*
|
|
* This kmem_cache is used for vmap_area objects. Instead of
|
|
* allocating from slab we reuse an object from this cache to
|
|
* make things faster. Especially in "no edge" splitting of
|
|
* free block.
|
|
*/
|
|
static struct kmem_cache *vmap_area_cachep;
|
|
|
|
/*
|
|
* This linked list is used in pair with free_vmap_area_root.
|
|
* It gives O(1) access to prev/next to perform fast coalescing.
|
|
*/
|
|
static LIST_HEAD(free_vmap_area_list);
|
|
|
|
/*
|
|
* This augment red-black tree represents the free vmap space.
|
|
* All vmap_area objects in this tree are sorted by va->va_start
|
|
* address. It is used for allocation and merging when a vmap
|
|
* object is released.
|
|
*
|
|
* Each vmap_area node contains a maximum available free block
|
|
* of its sub-tree, right or left. Therefore it is possible to
|
|
* find a lowest match of free area.
|
|
*/
|
|
static struct rb_root free_vmap_area_root = RB_ROOT;
|
|
|
|
static __always_inline unsigned long
|
|
va_size(struct vmap_area *va)
|
|
{
|
|
return (va->va_end - va->va_start);
|
|
}
|
|
|
|
static __always_inline unsigned long
|
|
get_subtree_max_size(struct rb_node *node)
|
|
{
|
|
struct vmap_area *va;
|
|
|
|
va = rb_entry_safe(node, struct vmap_area, rb_node);
|
|
return va ? va->subtree_max_size : 0;
|
|
}
|
|
|
|
/*
|
|
* Gets called when remove the node and rotate.
|
|
*/
|
|
static __always_inline unsigned long
|
|
compute_subtree_max_size(struct vmap_area *va)
|
|
{
|
|
return max3(va_size(va),
|
|
get_subtree_max_size(va->rb_node.rb_left),
|
|
get_subtree_max_size(va->rb_node.rb_right));
|
|
}
|
|
|
|
RB_DECLARE_CALLBACKS(static, free_vmap_area_rb_augment_cb,
|
|
struct vmap_area, rb_node, unsigned long, subtree_max_size,
|
|
compute_subtree_max_size)
|
|
|
|
static void purge_vmap_area_lazy(void);
|
|
static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
|
|
static unsigned long lazy_max_pages(void);
|
|
|
|
static atomic_long_t nr_vmalloc_pages;
|
|
|
|
unsigned long vmalloc_nr_pages(void)
|
|
{
|
|
return atomic_long_read(&nr_vmalloc_pages);
|
|
}
|
|
|
|
#ifdef CONFIG_ENABLE_VMALLOC_SAVING
|
|
#define POSSIBLE_VMALLOC_START PAGE_OFFSET
|
|
|
|
#define VMALLOC_BITMAP_SIZE ((VMALLOC_END - PAGE_OFFSET) >> \
|
|
PAGE_SHIFT)
|
|
#define VMALLOC_TO_BIT(addr) ((addr - PAGE_OFFSET) >> PAGE_SHIFT)
|
|
#define BIT_TO_VMALLOC(i) (PAGE_OFFSET + i * PAGE_SIZE)
|
|
|
|
unsigned long total_vmalloc_size;
|
|
unsigned long vmalloc_reserved;
|
|
|
|
DECLARE_BITMAP(possible_areas, VMALLOC_BITMAP_SIZE);
|
|
|
|
void mark_vmalloc_reserved_area(void *x, unsigned long size)
|
|
{
|
|
unsigned long addr = (unsigned long)x;
|
|
|
|
bitmap_set(possible_areas, VMALLOC_TO_BIT(addr), size >> PAGE_SHIFT);
|
|
vmalloc_reserved += size;
|
|
}
|
|
|
|
int is_vmalloc_addr(const void *x)
|
|
{
|
|
unsigned long addr = (unsigned long)x;
|
|
|
|
if (addr < POSSIBLE_VMALLOC_START || addr >= VMALLOC_END)
|
|
return 0;
|
|
|
|
if (test_bit(VMALLOC_TO_BIT(addr), possible_areas))
|
|
return 0;
|
|
|
|
return 1;
|
|
}
|
|
|
|
static void calc_total_vmalloc_size(void)
|
|
{
|
|
total_vmalloc_size = VMALLOC_END - POSSIBLE_VMALLOC_START -
|
|
vmalloc_reserved;
|
|
}
|
|
#else
|
|
int is_vmalloc_addr(const void *x)
|
|
{
|
|
unsigned long addr = (unsigned long)x;
|
|
|
|
return addr >= VMALLOC_START && addr < VMALLOC_END;
|
|
}
|
|
|
|
static void calc_total_vmalloc_size(void) { }
|
|
#endif
|
|
EXPORT_SYMBOL(is_vmalloc_addr);
|
|
|
|
static atomic_long_t nr_vmalloc_pages;
|
|
|
|
static struct vmap_area *__find_vmap_area(unsigned long addr)
|
|
{
|
|
struct rb_node *n = vmap_area_root.rb_node;
|
|
|
|
while (n) {
|
|
struct vmap_area *va;
|
|
|
|
va = rb_entry(n, struct vmap_area, rb_node);
|
|
if (addr < va->va_start)
|
|
n = n->rb_left;
|
|
else if (addr >= va->va_end)
|
|
n = n->rb_right;
|
|
else
|
|
return va;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
/*
|
|
* This function returns back addresses of parent node
|
|
* and its left or right link for further processing.
|
|
*/
|
|
static __always_inline struct rb_node **
|
|
find_va_links(struct vmap_area *va,
|
|
struct rb_root *root, struct rb_node *from,
|
|
struct rb_node **parent)
|
|
{
|
|
struct vmap_area *tmp_va;
|
|
struct rb_node **link;
|
|
|
|
if (root) {
|
|
link = &root->rb_node;
|
|
if (unlikely(!*link)) {
|
|
*parent = NULL;
|
|
return link;
|
|
}
|
|
} else {
|
|
link = &from;
|
|
}
|
|
|
|
/*
|
|
* Go to the bottom of the tree. When we hit the last point
|
|
* we end up with parent rb_node and correct direction, i name
|
|
* it link, where the new va->rb_node will be attached to.
|
|
*/
|
|
do {
|
|
tmp_va = rb_entry(*link, struct vmap_area, rb_node);
|
|
|
|
/*
|
|
* During the traversal we also do some sanity check.
|
|
* Trigger the BUG() if there are sides(left/right)
|
|
* or full overlaps.
|
|
*/
|
|
if (va->va_start < tmp_va->va_end &&
|
|
va->va_end <= tmp_va->va_start)
|
|
link = &(*link)->rb_left;
|
|
else if (va->va_end > tmp_va->va_start &&
|
|
va->va_start >= tmp_va->va_end)
|
|
link = &(*link)->rb_right;
|
|
else
|
|
BUG();
|
|
} while (*link);
|
|
|
|
*parent = &tmp_va->rb_node;
|
|
return link;
|
|
}
|
|
|
|
static __always_inline struct list_head *
|
|
get_va_next_sibling(struct rb_node *parent, struct rb_node **link)
|
|
{
|
|
struct list_head *list;
|
|
|
|
if (unlikely(!parent))
|
|
/*
|
|
* The red-black tree where we try to find VA neighbors
|
|
* before merging or inserting is empty, i.e. it means
|
|
* there is no free vmap space. Normally it does not
|
|
* happen but we handle this case anyway.
|
|
*/
|
|
return NULL;
|
|
|
|
list = &rb_entry(parent, struct vmap_area, rb_node)->list;
|
|
return (&parent->rb_right == link ? list->next : list);
|
|
}
|
|
|
|
static __always_inline void
|
|
link_va(struct vmap_area *va, struct rb_root *root,
|
|
struct rb_node *parent, struct rb_node **link, struct list_head *head)
|
|
{
|
|
/*
|
|
* VA is still not in the list, but we can
|
|
* identify its future previous list_head node.
|
|
*/
|
|
if (likely(parent)) {
|
|
head = &rb_entry(parent, struct vmap_area, rb_node)->list;
|
|
if (&parent->rb_right != link)
|
|
head = head->prev;
|
|
}
|
|
|
|
/* Insert to the rb-tree */
|
|
rb_link_node(&va->rb_node, parent, link);
|
|
if (root == &free_vmap_area_root) {
|
|
/*
|
|
* Some explanation here. Just perform simple insertion
|
|
* to the tree. We do not set va->subtree_max_size to
|
|
* its current size before calling rb_insert_augmented().
|
|
* It is because of we populate the tree from the bottom
|
|
* to parent levels when the node _is_ in the tree.
|
|
*
|
|
* Therefore we set subtree_max_size to zero after insertion,
|
|
* to let __augment_tree_propagate_from() puts everything to
|
|
* the correct order later on.
|
|
*/
|
|
rb_insert_augmented(&va->rb_node,
|
|
root, &free_vmap_area_rb_augment_cb);
|
|
va->subtree_max_size = 0;
|
|
} else {
|
|
rb_insert_color(&va->rb_node, root);
|
|
}
|
|
|
|
/* Address-sort this list */
|
|
list_add(&va->list, head);
|
|
}
|
|
|
|
static __always_inline void
|
|
unlink_va(struct vmap_area *va, struct rb_root *root)
|
|
{
|
|
/*
|
|
* During merging a VA node can be empty, therefore
|
|
* not linked with the tree nor list. Just check it.
|
|
*/
|
|
if (!RB_EMPTY_NODE(&va->rb_node)) {
|
|
if (root == &free_vmap_area_root)
|
|
rb_erase_augmented(&va->rb_node,
|
|
root, &free_vmap_area_rb_augment_cb);
|
|
else
|
|
rb_erase(&va->rb_node, root);
|
|
|
|
list_del(&va->list);
|
|
RB_CLEAR_NODE(&va->rb_node);
|
|
}
|
|
}
|
|
|
|
#if DEBUG_AUGMENT_PROPAGATE_CHECK
|
|
static void
|
|
augment_tree_propagate_check(struct rb_node *n)
|
|
{
|
|
struct vmap_area *va;
|
|
struct rb_node *node;
|
|
unsigned long size;
|
|
bool found = false;
|
|
|
|
if (n == NULL)
|
|
return;
|
|
|
|
va = rb_entry(n, struct vmap_area, rb_node);
|
|
size = va->subtree_max_size;
|
|
node = n;
|
|
|
|
while (node) {
|
|
va = rb_entry(node, struct vmap_area, rb_node);
|
|
|
|
if (get_subtree_max_size(node->rb_left) == size) {
|
|
node = node->rb_left;
|
|
} else {
|
|
if (va_size(va) == size) {
|
|
found = true;
|
|
break;
|
|
}
|
|
|
|
node = node->rb_right;
|
|
}
|
|
}
|
|
|
|
if (!found) {
|
|
va = rb_entry(n, struct vmap_area, rb_node);
|
|
pr_emerg("tree is corrupted: %lu, %lu\n",
|
|
va_size(va), va->subtree_max_size);
|
|
}
|
|
|
|
augment_tree_propagate_check(n->rb_left);
|
|
augment_tree_propagate_check(n->rb_right);
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* This function populates subtree_max_size from bottom to upper
|
|
* levels starting from VA point. The propagation must be done
|
|
* when VA size is modified by changing its va_start/va_end. Or
|
|
* in case of newly inserting of VA to the tree.
|
|
*
|
|
* It means that __augment_tree_propagate_from() must be called:
|
|
* - After VA has been inserted to the tree(free path);
|
|
* - After VA has been shrunk(allocation path);
|
|
* - After VA has been increased(merging path).
|
|
*
|
|
* Please note that, it does not mean that upper parent nodes
|
|
* and their subtree_max_size are recalculated all the time up
|
|
* to the root node.
|
|
*
|
|
* 4--8
|
|
* /\
|
|
* / \
|
|
* / \
|
|
* 2--2 8--8
|
|
*
|
|
* For example if we modify the node 4, shrinking it to 2, then
|
|
* no any modification is required. If we shrink the node 2 to 1
|
|
* its subtree_max_size is updated only, and set to 1. If we shrink
|
|
* the node 8 to 6, then its subtree_max_size is set to 6 and parent
|
|
* node becomes 4--6.
|
|
*/
|
|
static __always_inline void
|
|
augment_tree_propagate_from(struct vmap_area *va)
|
|
{
|
|
struct rb_node *node = &va->rb_node;
|
|
unsigned long new_va_sub_max_size;
|
|
|
|
while (node) {
|
|
va = rb_entry(node, struct vmap_area, rb_node);
|
|
new_va_sub_max_size = compute_subtree_max_size(va);
|
|
|
|
/*
|
|
* If the newly calculated maximum available size of the
|
|
* subtree is equal to the current one, then it means that
|
|
* the tree is propagated correctly. So we have to stop at
|
|
* this point to save cycles.
|
|
*/
|
|
if (va->subtree_max_size == new_va_sub_max_size)
|
|
break;
|
|
|
|
va->subtree_max_size = new_va_sub_max_size;
|
|
node = rb_parent(&va->rb_node);
|
|
}
|
|
|
|
#if DEBUG_AUGMENT_PROPAGATE_CHECK
|
|
augment_tree_propagate_check(free_vmap_area_root.rb_node);
|
|
#endif
|
|
}
|
|
|
|
static void
|
|
insert_vmap_area(struct vmap_area *va,
|
|
struct rb_root *root, struct list_head *head)
|
|
{
|
|
struct rb_node **link;
|
|
struct rb_node *parent;
|
|
|
|
link = find_va_links(va, root, NULL, &parent);
|
|
link_va(va, root, parent, link, head);
|
|
}
|
|
|
|
static void
|
|
insert_vmap_area_augment(struct vmap_area *va,
|
|
struct rb_node *from, struct rb_root *root,
|
|
struct list_head *head)
|
|
{
|
|
struct rb_node **link;
|
|
struct rb_node *parent;
|
|
|
|
if (from)
|
|
link = find_va_links(va, NULL, from, &parent);
|
|
else
|
|
link = find_va_links(va, root, NULL, &parent);
|
|
|
|
link_va(va, root, parent, link, head);
|
|
augment_tree_propagate_from(va);
|
|
}
|
|
|
|
/*
|
|
* Merge de-allocated chunk of VA memory with previous
|
|
* and next free blocks. If coalesce is not done a new
|
|
* free area is inserted. If VA has been merged, it is
|
|
* freed.
|
|
*/
|
|
static __always_inline void
|
|
merge_or_add_vmap_area(struct vmap_area *va,
|
|
struct rb_root *root, struct list_head *head)
|
|
{
|
|
struct vmap_area *sibling;
|
|
struct list_head *next;
|
|
struct rb_node **link;
|
|
struct rb_node *parent;
|
|
bool merged = false;
|
|
|
|
/*
|
|
* Find a place in the tree where VA potentially will be
|
|
* inserted, unless it is merged with its sibling/siblings.
|
|
*/
|
|
link = find_va_links(va, root, NULL, &parent);
|
|
|
|
/*
|
|
* Get next node of VA to check if merging can be done.
|
|
*/
|
|
next = get_va_next_sibling(parent, link);
|
|
if (unlikely(next == NULL))
|
|
goto insert;
|
|
|
|
/*
|
|
* start end
|
|
* | |
|
|
* |<------VA------>|<-----Next----->|
|
|
* | |
|
|
* start end
|
|
*/
|
|
if (next != head) {
|
|
sibling = list_entry(next, struct vmap_area, list);
|
|
if (sibling->va_start == va->va_end) {
|
|
sibling->va_start = va->va_start;
|
|
|
|
/* Check and update the tree if needed. */
|
|
augment_tree_propagate_from(sibling);
|
|
|
|
/* Remove this VA, it has been merged. */
|
|
unlink_va(va, root);
|
|
|
|
/* Free vmap_area object. */
|
|
kmem_cache_free(vmap_area_cachep, va);
|
|
|
|
/* Point to the new merged area. */
|
|
va = sibling;
|
|
merged = true;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* start end
|
|
* | |
|
|
* |<-----Prev----->|<------VA------>|
|
|
* | |
|
|
* start end
|
|
*/
|
|
if (next->prev != head) {
|
|
sibling = list_entry(next->prev, struct vmap_area, list);
|
|
if (sibling->va_end == va->va_start) {
|
|
sibling->va_end = va->va_end;
|
|
|
|
/* Check and update the tree if needed. */
|
|
augment_tree_propagate_from(sibling);
|
|
|
|
/* Remove this VA, it has been merged. */
|
|
unlink_va(va, root);
|
|
|
|
/* Free vmap_area object. */
|
|
kmem_cache_free(vmap_area_cachep, va);
|
|
|
|
return;
|
|
}
|
|
}
|
|
|
|
insert:
|
|
if (!merged) {
|
|
link_va(va, root, parent, link, head);
|
|
augment_tree_propagate_from(va);
|
|
}
|
|
}
|
|
|
|
static __always_inline bool
|
|
is_within_this_va(struct vmap_area *va, unsigned long size,
|
|
unsigned long align, unsigned long vstart)
|
|
{
|
|
unsigned long nva_start_addr;
|
|
|
|
if (va->va_start > vstart)
|
|
nva_start_addr = ALIGN(va->va_start, align);
|
|
else
|
|
nva_start_addr = ALIGN(vstart, align);
|
|
|
|
/* Can be overflowed due to big size or alignment. */
|
|
if (nva_start_addr + size < nva_start_addr ||
|
|
nva_start_addr < vstart)
|
|
return false;
|
|
|
|
return (nva_start_addr + size <= va->va_end);
|
|
}
|
|
|
|
/*
|
|
* Find the first free block(lowest start address) in the tree,
|
|
* that will accomplish the request corresponding to passing
|
|
* parameters.
|
|
*/
|
|
static __always_inline struct vmap_area *
|
|
find_vmap_lowest_match(unsigned long size,
|
|
unsigned long align, unsigned long vstart)
|
|
{
|
|
struct vmap_area *va;
|
|
struct rb_node *node;
|
|
unsigned long length;
|
|
|
|
/* Start from the root. */
|
|
node = free_vmap_area_root.rb_node;
|
|
|
|
/* Adjust the search size for alignment overhead. */
|
|
length = size + align - 1;
|
|
|
|
while (node) {
|
|
va = rb_entry(node, struct vmap_area, rb_node);
|
|
|
|
if (get_subtree_max_size(node->rb_left) >= length &&
|
|
vstart < va->va_start) {
|
|
node = node->rb_left;
|
|
} else {
|
|
if (is_within_this_va(va, size, align, vstart))
|
|
return va;
|
|
|
|
/*
|
|
* Does not make sense to go deeper towards the right
|
|
* sub-tree if it does not have a free block that is
|
|
* equal or bigger to the requested search length.
|
|
*/
|
|
if (get_subtree_max_size(node->rb_right) >= length) {
|
|
node = node->rb_right;
|
|
continue;
|
|
}
|
|
|
|
/*
|
|
* OK. We roll back and find the fist right sub-tree,
|
|
* that will satisfy the search criteria. It can happen
|
|
* only once due to "vstart" restriction.
|
|
*/
|
|
while ((node = rb_parent(node))) {
|
|
va = rb_entry(node, struct vmap_area, rb_node);
|
|
if (is_within_this_va(va, size, align, vstart))
|
|
return va;
|
|
|
|
if (get_subtree_max_size(node->rb_right) >= length &&
|
|
vstart <= va->va_start) {
|
|
node = node->rb_right;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
|
|
#include <linux/random.h>
|
|
|
|
static struct vmap_area *
|
|
find_vmap_lowest_linear_match(unsigned long size,
|
|
unsigned long align, unsigned long vstart)
|
|
{
|
|
struct vmap_area *va;
|
|
|
|
list_for_each_entry(va, &free_vmap_area_list, list) {
|
|
if (!is_within_this_va(va, size, align, vstart))
|
|
continue;
|
|
|
|
return va;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
static void
|
|
find_vmap_lowest_match_check(unsigned long size)
|
|
{
|
|
struct vmap_area *va_1, *va_2;
|
|
unsigned long vstart;
|
|
unsigned int rnd;
|
|
|
|
get_random_bytes(&rnd, sizeof(rnd));
|
|
vstart = VMALLOC_START + rnd;
|
|
|
|
va_1 = find_vmap_lowest_match(size, 1, vstart);
|
|
va_2 = find_vmap_lowest_linear_match(size, 1, vstart);
|
|
|
|
if (va_1 != va_2)
|
|
pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n",
|
|
va_1, va_2, vstart);
|
|
}
|
|
#endif
|
|
|
|
enum fit_type {
|
|
NOTHING_FIT = 0,
|
|
FL_FIT_TYPE = 1, /* full fit */
|
|
LE_FIT_TYPE = 2, /* left edge fit */
|
|
RE_FIT_TYPE = 3, /* right edge fit */
|
|
NE_FIT_TYPE = 4 /* no edge fit */
|
|
};
|
|
|
|
static __always_inline enum fit_type
|
|
classify_va_fit_type(struct vmap_area *va,
|
|
unsigned long nva_start_addr, unsigned long size)
|
|
{
|
|
enum fit_type type;
|
|
|
|
/* Check if it is within VA. */
|
|
if (nva_start_addr < va->va_start ||
|
|
nva_start_addr + size > va->va_end)
|
|
return NOTHING_FIT;
|
|
|
|
/* Now classify. */
|
|
if (va->va_start == nva_start_addr) {
|
|
if (va->va_end == nva_start_addr + size)
|
|
type = FL_FIT_TYPE;
|
|
else
|
|
type = LE_FIT_TYPE;
|
|
} else if (va->va_end == nva_start_addr + size) {
|
|
type = RE_FIT_TYPE;
|
|
} else {
|
|
type = NE_FIT_TYPE;
|
|
}
|
|
|
|
return type;
|
|
}
|
|
|
|
static __always_inline int
|
|
adjust_va_to_fit_type(struct vmap_area *va,
|
|
unsigned long nva_start_addr, unsigned long size,
|
|
enum fit_type type)
|
|
{
|
|
struct vmap_area *lva = NULL;
|
|
|
|
if (type == FL_FIT_TYPE) {
|
|
/*
|
|
* No need to split VA, it fully fits.
|
|
*
|
|
* | |
|
|
* V NVA V
|
|
* |---------------|
|
|
*/
|
|
unlink_va(va, &free_vmap_area_root);
|
|
kmem_cache_free(vmap_area_cachep, va);
|
|
} else if (type == LE_FIT_TYPE) {
|
|
/*
|
|
* Split left edge of fit VA.
|
|
*
|
|
* | |
|
|
* V NVA V R
|
|
* |-------|-------|
|
|
*/
|
|
va->va_start += size;
|
|
} else if (type == RE_FIT_TYPE) {
|
|
/*
|
|
* Split right edge of fit VA.
|
|
*
|
|
* | |
|
|
* L V NVA V
|
|
* |-------|-------|
|
|
*/
|
|
va->va_end = nva_start_addr;
|
|
} else if (type == NE_FIT_TYPE) {
|
|
/*
|
|
* Split no edge of fit VA.
|
|
*
|
|
* | |
|
|
* L V NVA V R
|
|
* |---|-------|---|
|
|
*/
|
|
lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
|
|
if (unlikely(!lva))
|
|
return -1;
|
|
|
|
/*
|
|
* Build the remainder.
|
|
*/
|
|
lva->va_start = va->va_start;
|
|
lva->va_end = nva_start_addr;
|
|
|
|
/*
|
|
* Shrink this VA to remaining size.
|
|
*/
|
|
va->va_start = nva_start_addr + size;
|
|
} else {
|
|
return -1;
|
|
}
|
|
|
|
if (type != FL_FIT_TYPE) {
|
|
augment_tree_propagate_from(va);
|
|
|
|
if (lva) /* type == NE_FIT_TYPE */
|
|
insert_vmap_area_augment(lva, &va->rb_node,
|
|
&free_vmap_area_root, &free_vmap_area_list);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Returns a start address of the newly allocated area, if success.
|
|
* Otherwise a vend is returned that indicates failure.
|
|
*/
|
|
static __always_inline unsigned long
|
|
__alloc_vmap_area(unsigned long size, unsigned long align,
|
|
unsigned long vstart, unsigned long vend, int node)
|
|
{
|
|
unsigned long nva_start_addr;
|
|
struct vmap_area *va;
|
|
enum fit_type type;
|
|
int ret;
|
|
|
|
va = find_vmap_lowest_match(size, align, vstart);
|
|
if (unlikely(!va))
|
|
return vend;
|
|
|
|
if (va->va_start > vstart)
|
|
nva_start_addr = ALIGN(va->va_start, align);
|
|
else
|
|
nva_start_addr = ALIGN(vstart, align);
|
|
|
|
/* Check the "vend" restriction. */
|
|
if (nva_start_addr + size > vend)
|
|
return vend;
|
|
|
|
/* Classify what we have found. */
|
|
type = classify_va_fit_type(va, nva_start_addr, size);
|
|
if (WARN_ON_ONCE(type == NOTHING_FIT))
|
|
return vend;
|
|
|
|
/* Update the free vmap_area. */
|
|
ret = adjust_va_to_fit_type(va, nva_start_addr, size, type);
|
|
if (ret)
|
|
return vend;
|
|
|
|
#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
|
|
find_vmap_lowest_match_check(size);
|
|
#endif
|
|
|
|
return nva_start_addr;
|
|
}
|
|
|
|
/*
|
|
* Allocate a region of KVA of the specified size and alignment, within the
|
|
* vstart and vend.
|
|
*/
|
|
static struct vmap_area *alloc_vmap_area(unsigned long size,
|
|
unsigned long align,
|
|
unsigned long vstart, unsigned long vend,
|
|
int node, gfp_t gfp_mask)
|
|
{
|
|
struct vmap_area *va;
|
|
unsigned long addr;
|
|
int purged = 0;
|
|
|
|
BUG_ON(!size);
|
|
BUG_ON(offset_in_page(size));
|
|
BUG_ON(!is_power_of_2(align));
|
|
|
|
if (unlikely(!vmap_initialized))
|
|
return ERR_PTR(-EBUSY);
|
|
|
|
might_sleep();
|
|
|
|
va = kmem_cache_alloc_node(vmap_area_cachep,
|
|
gfp_mask & GFP_RECLAIM_MASK, node);
|
|
if (unlikely(!va))
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
/*
|
|
* Only scan the relevant parts containing pointers to other objects
|
|
* to avoid false negatives.
|
|
*/
|
|
kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK);
|
|
|
|
retry:
|
|
spin_lock(&vmap_area_lock);
|
|
|
|
/*
|
|
* If an allocation fails, the "vend" address is
|
|
* returned. Therefore trigger the overflow path.
|
|
*/
|
|
addr = __alloc_vmap_area(size, align, vstart, vend, node);
|
|
if (unlikely(addr == vend))
|
|
goto overflow;
|
|
|
|
va->va_start = addr;
|
|
va->va_end = addr + size;
|
|
va->flags = 0;
|
|
insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
|
|
|
|
spin_unlock(&vmap_area_lock);
|
|
|
|
BUG_ON(!IS_ALIGNED(va->va_start, align));
|
|
BUG_ON(va->va_start < vstart);
|
|
BUG_ON(va->va_end > vend);
|
|
|
|
return va;
|
|
|
|
overflow:
|
|
spin_unlock(&vmap_area_lock);
|
|
if (!purged) {
|
|
purge_vmap_area_lazy();
|
|
purged = 1;
|
|
goto retry;
|
|
}
|
|
|
|
if (gfpflags_allow_blocking(gfp_mask)) {
|
|
unsigned long freed = 0;
|
|
blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
|
|
if (freed > 0) {
|
|
purged = 0;
|
|
goto retry;
|
|
}
|
|
}
|
|
|
|
if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit())
|
|
pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
|
|
size);
|
|
|
|
kmem_cache_free(vmap_area_cachep, va);
|
|
return ERR_PTR(-EBUSY);
|
|
}
|
|
|
|
int register_vmap_purge_notifier(struct notifier_block *nb)
|
|
{
|
|
return blocking_notifier_chain_register(&vmap_notify_list, nb);
|
|
}
|
|
EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
|
|
|
|
int unregister_vmap_purge_notifier(struct notifier_block *nb)
|
|
{
|
|
return blocking_notifier_chain_unregister(&vmap_notify_list, nb);
|
|
}
|
|
EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
|
|
|
|
static void __free_vmap_area(struct vmap_area *va)
|
|
{
|
|
BUG_ON(RB_EMPTY_NODE(&va->rb_node));
|
|
|
|
/*
|
|
* Remove from the busy tree/list.
|
|
*/
|
|
unlink_va(va, &vmap_area_root);
|
|
|
|
/*
|
|
* Merge VA with its neighbors, otherwise just add it.
|
|
*/
|
|
merge_or_add_vmap_area(va,
|
|
&free_vmap_area_root, &free_vmap_area_list);
|
|
}
|
|
|
|
/*
|
|
* Free a region of KVA allocated by alloc_vmap_area
|
|
*/
|
|
static void free_vmap_area(struct vmap_area *va)
|
|
{
|
|
spin_lock(&vmap_area_lock);
|
|
__free_vmap_area(va);
|
|
spin_unlock(&vmap_area_lock);
|
|
}
|
|
|
|
/*
|
|
* Clear the pagetable entries of a given vmap_area
|
|
*/
|
|
static void unmap_vmap_area(struct vmap_area *va)
|
|
{
|
|
vunmap_page_range(va->va_start, va->va_end);
|
|
}
|
|
|
|
/*
|
|
* lazy_max_pages is the maximum amount of virtual address space we gather up
|
|
* before attempting to purge with a TLB flush.
|
|
*
|
|
* There is a tradeoff here: a larger number will cover more kernel page tables
|
|
* and take slightly longer to purge, but it will linearly reduce the number of
|
|
* global TLB flushes that must be performed. It would seem natural to scale
|
|
* this number up linearly with the number of CPUs (because vmapping activity
|
|
* could also scale linearly with the number of CPUs), however it is likely
|
|
* that in practice, workloads might be constrained in other ways that mean
|
|
* vmap activity will not scale linearly with CPUs. Also, I want to be
|
|
* conservative and not introduce a big latency on huge systems, so go with
|
|
* a less aggressive log scale. It will still be an improvement over the old
|
|
* code, and it will be simple to change the scale factor if we find that it
|
|
* becomes a problem on bigger systems.
|
|
*/
|
|
static unsigned long lazy_max_pages(void)
|
|
{
|
|
unsigned int log;
|
|
|
|
log = fls(num_online_cpus());
|
|
|
|
return log * (1UL * CONFIG_VMAP_LAZY_PURGING_FACTOR *
|
|
1024 * 1024 / PAGE_SIZE);
|
|
}
|
|
|
|
static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0);
|
|
|
|
/*
|
|
* Serialize vmap purging. There is no actual criticial section protected
|
|
* by this look, but we want to avoid concurrent calls for performance
|
|
* reasons and to make the pcpu_get_vm_areas more deterministic.
|
|
*/
|
|
static DEFINE_MUTEX(vmap_purge_lock);
|
|
|
|
/* for per-CPU blocks */
|
|
static void purge_fragmented_blocks_allcpus(void);
|
|
|
|
/*
|
|
* called before a call to iounmap() if the caller wants vm_area_struct's
|
|
* immediately freed.
|
|
*/
|
|
void set_iounmap_nonlazy(void)
|
|
{
|
|
atomic_long_set(&vmap_lazy_nr, lazy_max_pages()+1);
|
|
}
|
|
|
|
/*
|
|
* Purges all lazily-freed vmap areas.
|
|
*/
|
|
static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end)
|
|
{
|
|
unsigned long resched_threshold;
|
|
struct llist_node *valist;
|
|
struct vmap_area *va;
|
|
struct vmap_area *n_va;
|
|
unsigned long flush_all_threshold = VMALLOC_END - VMALLOC_START;
|
|
|
|
lockdep_assert_held(&vmap_purge_lock);
|
|
|
|
valist = llist_del_all(&vmap_purge_list);
|
|
if (unlikely(valist == NULL))
|
|
return false;
|
|
|
|
/*
|
|
* TODO: to calculate a flush range without looping.
|
|
* The list can be up to lazy_max_pages() elements.
|
|
*/
|
|
llist_for_each_entry(va, valist, purge_list) {
|
|
if (va->va_start < start)
|
|
start = va->va_start;
|
|
if (va->va_end > end)
|
|
end = va->va_end;
|
|
}
|
|
|
|
if (end - start <= flush_all_threshold)
|
|
flush_tlb_kernel_range(start, end);
|
|
else
|
|
flush_tlb_all();
|
|
resched_threshold = lazy_max_pages() << 1;
|
|
|
|
spin_lock(&vmap_area_lock);
|
|
llist_for_each_entry_safe(va, n_va, valist, purge_list) {
|
|
unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT;
|
|
|
|
__free_vmap_area(va);
|
|
atomic_long_sub(nr, &vmap_lazy_nr);
|
|
|
|
if (atomic_long_read(&vmap_lazy_nr) < resched_threshold)
|
|
cond_resched_lock(&vmap_area_lock);
|
|
}
|
|
spin_unlock(&vmap_area_lock);
|
|
return true;
|
|
}
|
|
|
|
/*
|
|
* Kick off a purge of the outstanding lazy areas. Don't bother if somebody
|
|
* is already purging.
|
|
*/
|
|
static void try_purge_vmap_area_lazy(void)
|
|
{
|
|
if (mutex_trylock(&vmap_purge_lock)) {
|
|
__purge_vmap_area_lazy(ULONG_MAX, 0);
|
|
mutex_unlock(&vmap_purge_lock);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Kick off a purge of the outstanding lazy areas.
|
|
*/
|
|
static void purge_vmap_area_lazy(void)
|
|
{
|
|
mutex_lock(&vmap_purge_lock);
|
|
purge_fragmented_blocks_allcpus();
|
|
__purge_vmap_area_lazy(ULONG_MAX, 0);
|
|
mutex_unlock(&vmap_purge_lock);
|
|
}
|
|
|
|
/*
|
|
* Free a vmap area, caller ensuring that the area has been unmapped
|
|
* and flush_cache_vunmap had been called for the correct range
|
|
* previously.
|
|
*/
|
|
static void free_vmap_area_noflush(struct vmap_area *va)
|
|
{
|
|
unsigned long nr_lazy;
|
|
|
|
nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >>
|
|
PAGE_SHIFT, &vmap_lazy_nr);
|
|
|
|
/* After this point, we may free va at any time */
|
|
llist_add(&va->purge_list, &vmap_purge_list);
|
|
|
|
if (unlikely(nr_lazy > lazy_max_pages()))
|
|
try_purge_vmap_area_lazy();
|
|
}
|
|
|
|
/*
|
|
* Free and unmap a vmap area
|
|
*/
|
|
static void free_unmap_vmap_area(struct vmap_area *va)
|
|
{
|
|
flush_cache_vunmap(va->va_start, va->va_end);
|
|
unmap_vmap_area(va);
|
|
if (debug_pagealloc_enabled())
|
|
flush_tlb_kernel_range(va->va_start, va->va_end);
|
|
|
|
free_vmap_area_noflush(va);
|
|
}
|
|
|
|
static struct vmap_area *find_vmap_area(unsigned long addr)
|
|
{
|
|
struct vmap_area *va;
|
|
|
|
spin_lock(&vmap_area_lock);
|
|
va = __find_vmap_area(addr);
|
|
spin_unlock(&vmap_area_lock);
|
|
|
|
return va;
|
|
}
|
|
|
|
/*** Per cpu kva allocator ***/
|
|
|
|
/*
|
|
* vmap space is limited especially on 32 bit architectures. Ensure there is
|
|
* room for at least 16 percpu vmap blocks per CPU.
|
|
*/
|
|
/*
|
|
* If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
|
|
* to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
|
|
* instead (we just need a rough idea)
|
|
*/
|
|
#if BITS_PER_LONG == 32
|
|
#define VMALLOC_SPACE (128UL*1024*1024)
|
|
#else
|
|
#define VMALLOC_SPACE (128UL*1024*1024*1024)
|
|
#endif
|
|
|
|
#define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
|
|
#define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
|
|
#define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
|
|
#define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
|
|
#define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
|
|
#define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
|
|
#define VMAP_BBMAP_BITS \
|
|
VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
|
|
VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
|
|
VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
|
|
|
|
#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
|
|
|
|
struct vmap_block_queue {
|
|
spinlock_t lock;
|
|
struct list_head free;
|
|
};
|
|
|
|
struct vmap_block {
|
|
spinlock_t lock;
|
|
struct vmap_area *va;
|
|
unsigned long free, dirty;
|
|
unsigned long dirty_min, dirty_max; /*< dirty range */
|
|
struct list_head free_list;
|
|
struct rcu_head rcu_head;
|
|
struct list_head purge;
|
|
};
|
|
|
|
/* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
|
|
static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
|
|
|
|
/*
|
|
* Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
|
|
* in the free path. Could get rid of this if we change the API to return a
|
|
* "cookie" from alloc, to be passed to free. But no big deal yet.
|
|
*/
|
|
static DEFINE_SPINLOCK(vmap_block_tree_lock);
|
|
static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
|
|
|
|
/*
|
|
* We should probably have a fallback mechanism to allocate virtual memory
|
|
* out of partially filled vmap blocks. However vmap block sizing should be
|
|
* fairly reasonable according to the vmalloc size, so it shouldn't be a
|
|
* big problem.
|
|
*/
|
|
|
|
static unsigned long addr_to_vb_idx(unsigned long addr)
|
|
{
|
|
addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
|
|
addr /= VMAP_BLOCK_SIZE;
|
|
return addr;
|
|
}
|
|
|
|
static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off)
|
|
{
|
|
unsigned long addr;
|
|
|
|
addr = va_start + (pages_off << PAGE_SHIFT);
|
|
BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start));
|
|
return (void *)addr;
|
|
}
|
|
|
|
/**
|
|
* new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
|
|
* block. Of course pages number can't exceed VMAP_BBMAP_BITS
|
|
* @order: how many 2^order pages should be occupied in newly allocated block
|
|
* @gfp_mask: flags for the page level allocator
|
|
*
|
|
* Returns: virtual address in a newly allocated block or ERR_PTR(-errno)
|
|
*/
|
|
static void *new_vmap_block(unsigned int order, gfp_t gfp_mask)
|
|
{
|
|
struct vmap_block_queue *vbq;
|
|
struct vmap_block *vb;
|
|
struct vmap_area *va;
|
|
unsigned long vb_idx;
|
|
int node, err;
|
|
void *vaddr;
|
|
|
|
node = numa_node_id();
|
|
|
|
vb = kmalloc_node(sizeof(struct vmap_block),
|
|
gfp_mask & GFP_RECLAIM_MASK, node);
|
|
if (unlikely(!vb))
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
|
|
VMALLOC_START, VMALLOC_END,
|
|
node, gfp_mask);
|
|
if (IS_ERR(va)) {
|
|
kfree(vb);
|
|
return ERR_CAST(va);
|
|
}
|
|
|
|
err = radix_tree_preload(gfp_mask);
|
|
if (unlikely(err)) {
|
|
kfree(vb);
|
|
free_vmap_area(va);
|
|
return ERR_PTR(err);
|
|
}
|
|
|
|
vaddr = vmap_block_vaddr(va->va_start, 0);
|
|
spin_lock_init(&vb->lock);
|
|
vb->va = va;
|
|
/* At least something should be left free */
|
|
BUG_ON(VMAP_BBMAP_BITS <= (1UL << order));
|
|
vb->free = VMAP_BBMAP_BITS - (1UL << order);
|
|
vb->dirty = 0;
|
|
vb->dirty_min = VMAP_BBMAP_BITS;
|
|
vb->dirty_max = 0;
|
|
INIT_LIST_HEAD(&vb->free_list);
|
|
|
|
vb_idx = addr_to_vb_idx(va->va_start);
|
|
spin_lock(&vmap_block_tree_lock);
|
|
err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
|
|
spin_unlock(&vmap_block_tree_lock);
|
|
BUG_ON(err);
|
|
radix_tree_preload_end();
|
|
|
|
vbq = &get_cpu_var(vmap_block_queue);
|
|
spin_lock(&vbq->lock);
|
|
list_add_tail_rcu(&vb->free_list, &vbq->free);
|
|
spin_unlock(&vbq->lock);
|
|
put_cpu_var(vmap_block_queue);
|
|
|
|
return vaddr;
|
|
}
|
|
|
|
static void free_vmap_block(struct vmap_block *vb)
|
|
{
|
|
struct vmap_block *tmp;
|
|
unsigned long vb_idx;
|
|
|
|
vb_idx = addr_to_vb_idx(vb->va->va_start);
|
|
spin_lock(&vmap_block_tree_lock);
|
|
tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
|
|
spin_unlock(&vmap_block_tree_lock);
|
|
BUG_ON(tmp != vb);
|
|
|
|
free_vmap_area_noflush(vb->va);
|
|
kfree_rcu(vb, rcu_head);
|
|
}
|
|
|
|
static void purge_fragmented_blocks(int cpu)
|
|
{
|
|
LIST_HEAD(purge);
|
|
struct vmap_block *vb;
|
|
struct vmap_block *n_vb;
|
|
struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
|
|
|
|
rcu_read_lock();
|
|
list_for_each_entry_rcu(vb, &vbq->free, free_list) {
|
|
|
|
if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
|
|
continue;
|
|
|
|
spin_lock(&vb->lock);
|
|
if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
|
|
vb->free = 0; /* prevent further allocs after releasing lock */
|
|
vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
|
|
vb->dirty_min = 0;
|
|
vb->dirty_max = VMAP_BBMAP_BITS;
|
|
spin_lock(&vbq->lock);
|
|
list_del_rcu(&vb->free_list);
|
|
spin_unlock(&vbq->lock);
|
|
spin_unlock(&vb->lock);
|
|
list_add_tail(&vb->purge, &purge);
|
|
} else
|
|
spin_unlock(&vb->lock);
|
|
}
|
|
rcu_read_unlock();
|
|
|
|
list_for_each_entry_safe(vb, n_vb, &purge, purge) {
|
|
list_del(&vb->purge);
|
|
free_vmap_block(vb);
|
|
}
|
|
}
|
|
|
|
static void purge_fragmented_blocks_allcpus(void)
|
|
{
|
|
int cpu;
|
|
|
|
for_each_possible_cpu(cpu)
|
|
purge_fragmented_blocks(cpu);
|
|
}
|
|
|
|
static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
|
|
{
|
|
struct vmap_block_queue *vbq;
|
|
struct vmap_block *vb;
|
|
void *vaddr = NULL;
|
|
unsigned int order;
|
|
|
|
BUG_ON(offset_in_page(size));
|
|
BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
|
|
if (WARN_ON(size == 0)) {
|
|
/*
|
|
* Allocating 0 bytes isn't what caller wants since
|
|
* get_order(0) returns funny result. Just warn and terminate
|
|
* early.
|
|
*/
|
|
return NULL;
|
|
}
|
|
order = get_order(size);
|
|
|
|
rcu_read_lock();
|
|
vbq = &get_cpu_var(vmap_block_queue);
|
|
list_for_each_entry_rcu(vb, &vbq->free, free_list) {
|
|
unsigned long pages_off;
|
|
|
|
spin_lock(&vb->lock);
|
|
if (vb->free < (1UL << order)) {
|
|
spin_unlock(&vb->lock);
|
|
continue;
|
|
}
|
|
|
|
pages_off = VMAP_BBMAP_BITS - vb->free;
|
|
vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
|
|
vb->free -= 1UL << order;
|
|
if (vb->free == 0) {
|
|
spin_lock(&vbq->lock);
|
|
list_del_rcu(&vb->free_list);
|
|
spin_unlock(&vbq->lock);
|
|
}
|
|
|
|
spin_unlock(&vb->lock);
|
|
break;
|
|
}
|
|
|
|
put_cpu_var(vmap_block_queue);
|
|
rcu_read_unlock();
|
|
|
|
/* Allocate new block if nothing was found */
|
|
if (!vaddr)
|
|
vaddr = new_vmap_block(order, gfp_mask);
|
|
|
|
return vaddr;
|
|
}
|
|
|
|
static void vb_free(const void *addr, unsigned long size)
|
|
{
|
|
unsigned long offset;
|
|
unsigned long vb_idx;
|
|
unsigned int order;
|
|
struct vmap_block *vb;
|
|
|
|
BUG_ON(offset_in_page(size));
|
|
BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
|
|
|
|
flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
|
|
|
|
order = get_order(size);
|
|
|
|
offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
|
|
offset >>= PAGE_SHIFT;
|
|
|
|
vb_idx = addr_to_vb_idx((unsigned long)addr);
|
|
rcu_read_lock();
|
|
vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
|
|
rcu_read_unlock();
|
|
BUG_ON(!vb);
|
|
|
|
vunmap_page_range((unsigned long)addr, (unsigned long)addr + size);
|
|
|
|
if (debug_pagealloc_enabled())
|
|
flush_tlb_kernel_range((unsigned long)addr,
|
|
(unsigned long)addr + size);
|
|
|
|
spin_lock(&vb->lock);
|
|
|
|
/* Expand dirty range */
|
|
vb->dirty_min = min(vb->dirty_min, offset);
|
|
vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
|
|
|
|
vb->dirty += 1UL << order;
|
|
if (vb->dirty == VMAP_BBMAP_BITS) {
|
|
BUG_ON(vb->free);
|
|
spin_unlock(&vb->lock);
|
|
free_vmap_block(vb);
|
|
} else
|
|
spin_unlock(&vb->lock);
|
|
}
|
|
|
|
/**
|
|
* vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
|
|
*
|
|
* The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
|
|
* to amortize TLB flushing overheads. What this means is that any page you
|
|
* have now, may, in a former life, have been mapped into kernel virtual
|
|
* address by the vmap layer and so there might be some CPUs with TLB entries
|
|
* still referencing that page (additional to the regular 1:1 kernel mapping).
|
|
*
|
|
* vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
|
|
* be sure that none of the pages we have control over will have any aliases
|
|
* from the vmap layer.
|
|
*/
|
|
void vm_unmap_aliases(void)
|
|
{
|
|
unsigned long start = ULONG_MAX, end = 0;
|
|
int cpu;
|
|
int flush = 0;
|
|
|
|
if (unlikely(!vmap_initialized))
|
|
return;
|
|
|
|
might_sleep();
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
|
|
struct vmap_block *vb;
|
|
|
|
rcu_read_lock();
|
|
list_for_each_entry_rcu(vb, &vbq->free, free_list) {
|
|
spin_lock(&vb->lock);
|
|
if (vb->dirty) {
|
|
unsigned long va_start = vb->va->va_start;
|
|
unsigned long s, e;
|
|
|
|
s = va_start + (vb->dirty_min << PAGE_SHIFT);
|
|
e = va_start + (vb->dirty_max << PAGE_SHIFT);
|
|
|
|
start = min(s, start);
|
|
end = max(e, end);
|
|
|
|
flush = 1;
|
|
}
|
|
spin_unlock(&vb->lock);
|
|
}
|
|
rcu_read_unlock();
|
|
}
|
|
|
|
mutex_lock(&vmap_purge_lock);
|
|
purge_fragmented_blocks_allcpus();
|
|
if (!__purge_vmap_area_lazy(start, end) && flush)
|
|
flush_tlb_kernel_range(start, end);
|
|
mutex_unlock(&vmap_purge_lock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(vm_unmap_aliases);
|
|
|
|
/**
|
|
* vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
|
|
* @mem: the pointer returned by vm_map_ram
|
|
* @count: the count passed to that vm_map_ram call (cannot unmap partial)
|
|
*/
|
|
void vm_unmap_ram(const void *mem, unsigned int count)
|
|
{
|
|
unsigned long size = (unsigned long)count << PAGE_SHIFT;
|
|
unsigned long addr = (unsigned long)mem;
|
|
struct vmap_area *va;
|
|
|
|
might_sleep();
|
|
BUG_ON(!addr);
|
|
BUG_ON(addr < VMALLOC_START);
|
|
BUG_ON(addr > VMALLOC_END);
|
|
BUG_ON(!PAGE_ALIGNED(addr));
|
|
|
|
if (likely(count <= VMAP_MAX_ALLOC)) {
|
|
debug_check_no_locks_freed(mem, size);
|
|
vb_free(mem, size);
|
|
return;
|
|
}
|
|
|
|
va = find_vmap_area(addr);
|
|
BUG_ON(!va);
|
|
debug_check_no_locks_freed((void *)va->va_start,
|
|
(va->va_end - va->va_start));
|
|
free_unmap_vmap_area(va);
|
|
}
|
|
EXPORT_SYMBOL(vm_unmap_ram);
|
|
|
|
/**
|
|
* vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
|
|
* @pages: an array of pointers to the pages to be mapped
|
|
* @count: number of pages
|
|
* @node: prefer to allocate data structures on this node
|
|
* @prot: memory protection to use. PAGE_KERNEL for regular RAM
|
|
*
|
|
* If you use this function for less than VMAP_MAX_ALLOC pages, it could be
|
|
* faster than vmap so it's good. But if you mix long-life and short-life
|
|
* objects with vm_map_ram(), it could consume lots of address space through
|
|
* fragmentation (especially on a 32bit machine). You could see failures in
|
|
* the end. Please use this function for short-lived objects.
|
|
*
|
|
* Returns: a pointer to the address that has been mapped, or %NULL on failure
|
|
*/
|
|
void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
|
|
{
|
|
unsigned long size = (unsigned long)count << PAGE_SHIFT;
|
|
unsigned long addr;
|
|
void *mem;
|
|
|
|
if (likely(count <= VMAP_MAX_ALLOC)) {
|
|
mem = vb_alloc(size, GFP_KERNEL);
|
|
if (IS_ERR(mem))
|
|
return NULL;
|
|
addr = (unsigned long)mem;
|
|
} else {
|
|
struct vmap_area *va;
|
|
va = alloc_vmap_area(size, PAGE_SIZE,
|
|
VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
|
|
if (IS_ERR(va))
|
|
return NULL;
|
|
|
|
addr = va->va_start;
|
|
mem = (void *)addr;
|
|
}
|
|
if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
|
|
vm_unmap_ram(mem, count);
|
|
return NULL;
|
|
}
|
|
return mem;
|
|
}
|
|
EXPORT_SYMBOL(vm_map_ram);
|
|
|
|
static struct vm_struct *vmlist __initdata;
|
|
|
|
/**
|
|
* vm_area_check_early - check if vmap area is already mapped
|
|
* @vm: vm_struct to be checked
|
|
*
|
|
* This function is used to check if the vmap area has been
|
|
* mapped already. @vm->addr, @vm->size and @vm->flags should
|
|
* contain proper values.
|
|
*
|
|
*/
|
|
int __init vm_area_check_early(struct vm_struct *vm)
|
|
{
|
|
struct vm_struct *tmp, **p;
|
|
|
|
BUG_ON(vmap_initialized);
|
|
for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
|
|
if (tmp->addr >= vm->addr) {
|
|
if (tmp->addr < vm->addr + vm->size)
|
|
return 1;
|
|
} else {
|
|
if (tmp->addr + tmp->size > vm->addr)
|
|
return 1;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* vm_area_add_early - add vmap area early during boot
|
|
* @vm: vm_struct to add
|
|
*
|
|
* This function is used to add fixed kernel vm area to vmlist before
|
|
* vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
|
|
* should contain proper values and the other fields should be zero.
|
|
*
|
|
* DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
|
|
*/
|
|
void __init vm_area_add_early(struct vm_struct *vm)
|
|
{
|
|
struct vm_struct *tmp, **p;
|
|
|
|
BUG_ON(vmap_initialized);
|
|
for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
|
|
if (tmp->addr >= vm->addr) {
|
|
BUG_ON(tmp->addr < vm->addr + vm->size);
|
|
break;
|
|
} else
|
|
BUG_ON(tmp->addr + tmp->size > vm->addr);
|
|
}
|
|
vm->next = *p;
|
|
*p = vm;
|
|
}
|
|
|
|
/**
|
|
* vm_area_register_early - register vmap area early during boot
|
|
* @vm: vm_struct to register
|
|
* @align: requested alignment
|
|
*
|
|
* This function is used to register kernel vm area before
|
|
* vmalloc_init() is called. @vm->size and @vm->flags should contain
|
|
* proper values on entry and other fields should be zero. On return,
|
|
* vm->addr contains the allocated address.
|
|
*
|
|
* DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
|
|
*/
|
|
void __init vm_area_register_early(struct vm_struct *vm, size_t align)
|
|
{
|
|
static size_t vm_init_off __initdata;
|
|
unsigned long addr;
|
|
|
|
addr = ALIGN(VMALLOC_START + vm_init_off, align);
|
|
vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
|
|
|
|
vm->addr = (void *)addr;
|
|
|
|
vm_area_add_early(vm);
|
|
}
|
|
|
|
static void vmap_init_free_space(void)
|
|
{
|
|
unsigned long vmap_start = 1;
|
|
const unsigned long vmap_end = ULONG_MAX;
|
|
struct vmap_area *busy, *free;
|
|
|
|
/*
|
|
* B F B B B F
|
|
* -|-----|.....|-----|-----|-----|.....|-
|
|
* | The KVA space |
|
|
* |<--------------------------------->|
|
|
*/
|
|
list_for_each_entry(busy, &vmap_area_list, list) {
|
|
if (busy->va_start - vmap_start > 0) {
|
|
free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
|
|
if (!WARN_ON_ONCE(!free)) {
|
|
free->va_start = vmap_start;
|
|
free->va_end = busy->va_start;
|
|
|
|
insert_vmap_area_augment(free, NULL,
|
|
&free_vmap_area_root,
|
|
&free_vmap_area_list);
|
|
}
|
|
}
|
|
|
|
vmap_start = busy->va_end;
|
|
}
|
|
|
|
if (vmap_end - vmap_start > 0) {
|
|
free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
|
|
if (!WARN_ON_ONCE(!free)) {
|
|
free->va_start = vmap_start;
|
|
free->va_end = vmap_end;
|
|
|
|
insert_vmap_area_augment(free, NULL,
|
|
&free_vmap_area_root,
|
|
&free_vmap_area_list);
|
|
}
|
|
}
|
|
}
|
|
|
|
void __init vmalloc_init(void)
|
|
{
|
|
struct vmap_area *va;
|
|
struct vm_struct *tmp;
|
|
int i;
|
|
|
|
/*
|
|
* Create the cache for vmap_area objects.
|
|
*/
|
|
vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC);
|
|
|
|
for_each_possible_cpu(i) {
|
|
struct vmap_block_queue *vbq;
|
|
struct vfree_deferred *p;
|
|
|
|
vbq = &per_cpu(vmap_block_queue, i);
|
|
spin_lock_init(&vbq->lock);
|
|
INIT_LIST_HEAD(&vbq->free);
|
|
p = &per_cpu(vfree_deferred, i);
|
|
init_llist_head(&p->list);
|
|
INIT_WORK(&p->wq, free_work);
|
|
}
|
|
|
|
/* Import existing vmlist entries. */
|
|
for (tmp = vmlist; tmp; tmp = tmp->next) {
|
|
va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
|
|
if (WARN_ON_ONCE(!va))
|
|
continue;
|
|
|
|
va->flags = VM_VM_AREA;
|
|
va->va_start = (unsigned long)tmp->addr;
|
|
va->va_end = va->va_start + tmp->size;
|
|
va->vm = tmp;
|
|
insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
|
|
}
|
|
|
|
/*
|
|
* Now we can initialize a free vmap space.
|
|
*/
|
|
vmap_init_free_space();
|
|
calc_total_vmalloc_size();
|
|
vmap_initialized = true;
|
|
}
|
|
|
|
/**
|
|
* map_kernel_range_noflush - map kernel VM area with the specified pages
|
|
* @addr: start of the VM area to map
|
|
* @size: size of the VM area to map
|
|
* @prot: page protection flags to use
|
|
* @pages: pages to map
|
|
*
|
|
* Map PFN_UP(@size) pages at @addr. The VM area @addr and @size
|
|
* specify should have been allocated using get_vm_area() and its
|
|
* friends.
|
|
*
|
|
* NOTE:
|
|
* This function does NOT do any cache flushing. The caller is
|
|
* responsible for calling flush_cache_vmap() on to-be-mapped areas
|
|
* before calling this function.
|
|
*
|
|
* RETURNS:
|
|
* The number of pages mapped on success, -errno on failure.
|
|
*/
|
|
int map_kernel_range_noflush(unsigned long addr, unsigned long size,
|
|
pgprot_t prot, struct page **pages)
|
|
{
|
|
return vmap_page_range_noflush(addr, addr + size, prot, pages);
|
|
}
|
|
|
|
/**
|
|
* unmap_kernel_range_noflush - unmap kernel VM area
|
|
* @addr: start of the VM area to unmap
|
|
* @size: size of the VM area to unmap
|
|
*
|
|
* Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size
|
|
* specify should have been allocated using get_vm_area() and its
|
|
* friends.
|
|
*
|
|
* NOTE:
|
|
* This function does NOT do any cache flushing. The caller is
|
|
* responsible for calling flush_cache_vunmap() on to-be-mapped areas
|
|
* before calling this function and flush_tlb_kernel_range() after.
|
|
*/
|
|
void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
|
|
{
|
|
vunmap_page_range(addr, addr + size);
|
|
}
|
|
EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush);
|
|
|
|
/**
|
|
* unmap_kernel_range - unmap kernel VM area and flush cache and TLB
|
|
* @addr: start of the VM area to unmap
|
|
* @size: size of the VM area to unmap
|
|
*
|
|
* Similar to unmap_kernel_range_noflush() but flushes vcache before
|
|
* the unmapping and tlb after.
|
|
*/
|
|
void unmap_kernel_range(unsigned long addr, unsigned long size)
|
|
{
|
|
unsigned long end = addr + size;
|
|
|
|
flush_cache_vunmap(addr, end);
|
|
vunmap_page_range(addr, end);
|
|
flush_tlb_kernel_range(addr, end);
|
|
}
|
|
EXPORT_SYMBOL_GPL(unmap_kernel_range);
|
|
|
|
int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages)
|
|
{
|
|
unsigned long addr = (unsigned long)area->addr;
|
|
unsigned long end = addr + get_vm_area_size(area);
|
|
int err;
|
|
|
|
err = vmap_page_range(addr, end, prot, pages);
|
|
|
|
return err > 0 ? 0 : err;
|
|
}
|
|
EXPORT_SYMBOL_GPL(map_vm_area);
|
|
|
|
static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
|
|
unsigned long flags, const void *caller)
|
|
{
|
|
spin_lock(&vmap_area_lock);
|
|
vm->flags = flags;
|
|
vm->addr = (void *)va->va_start;
|
|
vm->size = va->va_end - va->va_start;
|
|
vm->caller = caller;
|
|
va->vm = vm;
|
|
va->flags |= VM_VM_AREA;
|
|
spin_unlock(&vmap_area_lock);
|
|
}
|
|
|
|
static void clear_vm_uninitialized_flag(struct vm_struct *vm)
|
|
{
|
|
/*
|
|
* Before removing VM_UNINITIALIZED,
|
|
* we should make sure that vm has proper values.
|
|
* Pair with smp_rmb() in show_numa_info().
|
|
*/
|
|
smp_wmb();
|
|
vm->flags &= ~VM_UNINITIALIZED;
|
|
}
|
|
|
|
static struct vm_struct *__get_vm_area_node(unsigned long size,
|
|
unsigned long align, unsigned long flags, unsigned long start,
|
|
unsigned long end, int node, gfp_t gfp_mask, const void *caller)
|
|
{
|
|
struct vmap_area *va;
|
|
struct vm_struct *area;
|
|
|
|
BUG_ON(in_interrupt());
|
|
size = PAGE_ALIGN(size);
|
|
if (unlikely(!size))
|
|
return NULL;
|
|
|
|
if (flags & VM_IOREMAP)
|
|
align = 1ul << clamp_t(int, get_count_order_long(size),
|
|
PAGE_SHIFT, IOREMAP_MAX_ORDER);
|
|
|
|
area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
|
|
if (unlikely(!area))
|
|
return NULL;
|
|
|
|
if (!(flags & VM_NO_GUARD))
|
|
size += PAGE_SIZE;
|
|
|
|
va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
|
|
if (IS_ERR(va)) {
|
|
kfree(area);
|
|
return NULL;
|
|
}
|
|
|
|
setup_vmalloc_vm(area, va, flags, caller);
|
|
|
|
return area;
|
|
}
|
|
|
|
struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
|
|
unsigned long start, unsigned long end)
|
|
{
|
|
return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
|
|
GFP_KERNEL, __builtin_return_address(0));
|
|
}
|
|
EXPORT_SYMBOL_GPL(__get_vm_area);
|
|
|
|
struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
|
|
unsigned long start, unsigned long end,
|
|
const void *caller)
|
|
{
|
|
return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
|
|
GFP_KERNEL, caller);
|
|
}
|
|
|
|
/**
|
|
* get_vm_area - reserve a contiguous kernel virtual area
|
|
* @size: size of the area
|
|
* @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
|
|
*
|
|
* Search an area of @size in the kernel virtual mapping area,
|
|
* and reserved it for out purposes. Returns the area descriptor
|
|
* on success or %NULL on failure.
|
|
*/
|
|
struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
|
|
{
|
|
#ifdef CONFIG_ENABLE_VMALLOC_SAVING
|
|
return __get_vm_area_node(size, 1, flags, PAGE_OFFSET, VMALLOC_END,
|
|
NUMA_NO_NODE, GFP_KERNEL,
|
|
__builtin_return_address(0));
|
|
#else
|
|
return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
|
|
NUMA_NO_NODE, GFP_KERNEL,
|
|
__builtin_return_address(0));
|
|
#endif
|
|
}
|
|
EXPORT_SYMBOL_GPL(get_vm_area);
|
|
|
|
struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
|
|
const void *caller)
|
|
{
|
|
#ifdef CONFIG_ENABLE_VMALLOC_SAVING
|
|
return __get_vm_area_node(size, 1, flags, PAGE_OFFSET, VMALLOC_END,
|
|
NUMA_NO_NODE, GFP_KERNEL, caller);
|
|
#else
|
|
return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
|
|
NUMA_NO_NODE, GFP_KERNEL, caller);
|
|
#endif
|
|
}
|
|
|
|
/**
|
|
* find_vm_area - find a continuous kernel virtual area
|
|
* @addr: base address
|
|
*
|
|
* Search for the kernel VM area starting at @addr, and return it.
|
|
* It is up to the caller to do all required locking to keep the returned
|
|
* pointer valid.
|
|
*/
|
|
struct vm_struct *find_vm_area(const void *addr)
|
|
{
|
|
struct vmap_area *va;
|
|
|
|
va = find_vmap_area((unsigned long)addr);
|
|
if (va && va->flags & VM_VM_AREA)
|
|
return va->vm;
|
|
|
|
return NULL;
|
|
}
|
|
|
|
static struct vm_struct *__remove_vm_area(struct vmap_area *va)
|
|
{
|
|
struct vm_struct *vm = va->vm;
|
|
|
|
spin_lock(&vmap_area_lock);
|
|
va->vm = NULL;
|
|
va->flags &= ~VM_VM_AREA;
|
|
va->flags |= VM_LAZY_FREE;
|
|
spin_unlock(&vmap_area_lock);
|
|
|
|
kasan_free_shadow(vm);
|
|
free_unmap_vmap_area(va);
|
|
|
|
return vm;
|
|
}
|
|
|
|
/**
|
|
* remove_vm_area - find and remove a continuous kernel virtual area
|
|
* @addr: base address
|
|
*
|
|
* Search for the kernel VM area starting at @addr, and remove it.
|
|
* This function returns the found VM area, but using it is NOT safe
|
|
* on SMP machines, except for its size or flags.
|
|
*/
|
|
struct vm_struct *remove_vm_area(const void *addr)
|
|
{
|
|
struct vm_struct *vm = NULL;
|
|
struct vmap_area *va;
|
|
|
|
va = find_vmap_area((unsigned long)addr);
|
|
if (va && va->flags & VM_VM_AREA)
|
|
vm = __remove_vm_area(va);
|
|
|
|
return vm;
|
|
}
|
|
|
|
static void __vunmap(const void *addr, int deallocate_pages)
|
|
{
|
|
struct vm_struct *area;
|
|
struct vmap_area *va;
|
|
|
|
if (!addr)
|
|
return;
|
|
|
|
if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
|
|
addr))
|
|
return;
|
|
|
|
va = find_vmap_area((unsigned long)addr);
|
|
if (unlikely(!va || !(va->flags & VM_VM_AREA))) {
|
|
WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
|
|
addr);
|
|
return;
|
|
}
|
|
|
|
area = va->vm;
|
|
debug_check_no_locks_freed(addr, get_vm_area_size(area));
|
|
debug_check_no_obj_freed(addr, get_vm_area_size(area));
|
|
|
|
__remove_vm_area(va);
|
|
if (deallocate_pages) {
|
|
int i;
|
|
|
|
for (i = 0; i < area->nr_pages; i++) {
|
|
struct page *page = area->pages[i];
|
|
|
|
BUG_ON(!page);
|
|
__free_pages(page, 0);
|
|
}
|
|
atomic_long_sub(area->nr_pages, &nr_vmalloc_pages);
|
|
|
|
kvfree(area->pages);
|
|
}
|
|
|
|
kfree(area);
|
|
}
|
|
|
|
static inline void __vfree_deferred(const void *addr)
|
|
{
|
|
/*
|
|
* Use raw_cpu_ptr() because this can be called from preemptible
|
|
* context. Preemption is absolutely fine here, because the llist_add()
|
|
* implementation is lockless, so it works even if we are adding to
|
|
* nother cpu's list. schedule_work() should be fine with this too.
|
|
*/
|
|
struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred);
|
|
|
|
if (llist_add((struct llist_node *)addr, &p->list))
|
|
schedule_work(&p->wq);
|
|
}
|
|
|
|
/**
|
|
* vfree_atomic - release memory allocated by vmalloc()
|
|
* @addr: memory base address
|
|
*
|
|
* This one is just like vfree() but can be called in any atomic context
|
|
* except NMIs.
|
|
*/
|
|
void vfree_atomic(const void *addr)
|
|
{
|
|
BUG_ON(in_nmi());
|
|
|
|
kmemleak_free(addr);
|
|
|
|
if (!addr)
|
|
return;
|
|
__vfree_deferred(addr);
|
|
}
|
|
|
|
/**
|
|
* vfree - release memory allocated by vmalloc()
|
|
* @addr: memory base address
|
|
*
|
|
* Free the virtually continuous memory area starting at @addr, as
|
|
* obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
|
|
* NULL, no operation is performed.
|
|
*
|
|
* Must not be called in NMI context (strictly speaking, only if we don't
|
|
* have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
|
|
* conventions for vfree() arch-depenedent would be a really bad idea)
|
|
*
|
|
* NOTE: assumes that the object at @addr has a size >= sizeof(llist_node)
|
|
*/
|
|
void vfree(const void *addr)
|
|
{
|
|
BUG_ON(in_nmi());
|
|
|
|
kmemleak_free(addr);
|
|
|
|
if (!addr)
|
|
return;
|
|
if (unlikely(in_interrupt()))
|
|
__vfree_deferred(addr);
|
|
else
|
|
__vunmap(addr, 1);
|
|
}
|
|
EXPORT_SYMBOL(vfree);
|
|
|
|
/**
|
|
* vunmap - release virtual mapping obtained by vmap()
|
|
* @addr: memory base address
|
|
*
|
|
* Free the virtually contiguous memory area starting at @addr,
|
|
* which was created from the page array passed to vmap().
|
|
*
|
|
* Must not be called in interrupt context.
|
|
*/
|
|
void vunmap(const void *addr)
|
|
{
|
|
BUG_ON(in_interrupt());
|
|
might_sleep();
|
|
if (addr)
|
|
__vunmap(addr, 0);
|
|
}
|
|
EXPORT_SYMBOL(vunmap);
|
|
|
|
/**
|
|
* vmap - map an array of pages into virtually contiguous space
|
|
* @pages: array of page pointers
|
|
* @count: number of pages to map
|
|
* @flags: vm_area->flags
|
|
* @prot: page protection for the mapping
|
|
*
|
|
* Maps @count pages from @pages into contiguous kernel virtual
|
|
* space.
|
|
*/
|
|
void *vmap(struct page **pages, unsigned int count,
|
|
unsigned long flags, pgprot_t prot)
|
|
{
|
|
struct vm_struct *area;
|
|
unsigned long size; /* In bytes */
|
|
|
|
might_sleep();
|
|
|
|
if (count > totalram_pages)
|
|
return NULL;
|
|
|
|
size = (unsigned long)count << PAGE_SHIFT;
|
|
area = get_vm_area_caller(size, flags, __builtin_return_address(0));
|
|
if (!area)
|
|
return NULL;
|
|
|
|
if (map_vm_area(area, prot, pages)) {
|
|
vunmap(area->addr);
|
|
return NULL;
|
|
}
|
|
|
|
return area->addr;
|
|
}
|
|
EXPORT_SYMBOL(vmap);
|
|
|
|
static void *__vmalloc_node(unsigned long size, unsigned long align,
|
|
gfp_t gfp_mask, pgprot_t prot,
|
|
int node, const void *caller);
|
|
static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
|
|
pgprot_t prot, int node)
|
|
{
|
|
struct page **pages;
|
|
unsigned int nr_pages, array_size, i;
|
|
const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
|
|
const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN;
|
|
const gfp_t highmem_mask = (gfp_mask & (GFP_DMA | GFP_DMA32)) ?
|
|
0 :
|
|
__GFP_HIGHMEM;
|
|
|
|
nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
|
|
array_size = (nr_pages * sizeof(struct page *));
|
|
|
|
/* Please note that the recursion is strictly bounded. */
|
|
if (array_size > PAGE_SIZE) {
|
|
pages = __vmalloc_node(array_size, 1, nested_gfp|highmem_mask,
|
|
PAGE_KERNEL, node, area->caller);
|
|
} else {
|
|
pages = kmalloc_node(array_size, nested_gfp, node);
|
|
}
|
|
|
|
if (!pages) {
|
|
remove_vm_area(area->addr);
|
|
kfree(area);
|
|
return NULL;
|
|
}
|
|
|
|
area->pages = pages;
|
|
area->nr_pages = nr_pages;
|
|
|
|
for (i = 0; i < area->nr_pages; i++) {
|
|
struct page *page;
|
|
|
|
if (node == NUMA_NO_NODE)
|
|
page = alloc_page(alloc_mask|highmem_mask);
|
|
else
|
|
page = alloc_pages_node(node, alloc_mask|highmem_mask, 0);
|
|
|
|
if (unlikely(!page)) {
|
|
/* Successfully allocated i pages, free them in __vunmap() */
|
|
area->nr_pages = i;
|
|
atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
|
|
goto fail;
|
|
}
|
|
area->pages[i] = page;
|
|
if (gfpflags_allow_blocking(gfp_mask|highmem_mask))
|
|
cond_resched();
|
|
}
|
|
atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
|
|
|
|
if (map_vm_area(area, prot, pages))
|
|
goto fail;
|
|
return area->addr;
|
|
|
|
fail:
|
|
warn_alloc(gfp_mask, NULL,
|
|
"vmalloc: allocation failure, allocated %ld of %ld bytes",
|
|
(area->nr_pages*PAGE_SIZE), area->size);
|
|
vfree(area->addr);
|
|
return NULL;
|
|
}
|
|
|
|
/**
|
|
* __vmalloc_node_range - allocate virtually contiguous memory
|
|
* @size: allocation size
|
|
* @align: desired alignment
|
|
* @start: vm area range start
|
|
* @end: vm area range end
|
|
* @gfp_mask: flags for the page level allocator
|
|
* @prot: protection mask for the allocated pages
|
|
* @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD)
|
|
* @node: node to use for allocation or NUMA_NO_NODE
|
|
* @caller: caller's return address
|
|
*
|
|
* Allocate enough pages to cover @size from the page level
|
|
* allocator with @gfp_mask flags. Map them into contiguous
|
|
* kernel virtual space, using a pagetable protection of @prot.
|
|
*/
|
|
void *__vmalloc_node_range(unsigned long size, unsigned long align,
|
|
unsigned long start, unsigned long end, gfp_t gfp_mask,
|
|
pgprot_t prot, unsigned long vm_flags, int node,
|
|
const void *caller)
|
|
{
|
|
struct vm_struct *area;
|
|
void *addr;
|
|
unsigned long real_size = size;
|
|
|
|
size = PAGE_ALIGN(size);
|
|
if (!size || (size >> PAGE_SHIFT) > totalram_pages)
|
|
goto fail;
|
|
|
|
area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED |
|
|
vm_flags, start, end, node, gfp_mask, caller);
|
|
if (!area)
|
|
goto fail;
|
|
|
|
addr = __vmalloc_area_node(area, gfp_mask, prot, node);
|
|
if (!addr)
|
|
return NULL;
|
|
|
|
/*
|
|
* First make sure the mappings are removed from all page-tables
|
|
* before they are freed.
|
|
*/
|
|
vmalloc_sync_unmappings();
|
|
|
|
/*
|
|
* In this function, newly allocated vm_struct has VM_UNINITIALIZED
|
|
* flag. It means that vm_struct is not fully initialized.
|
|
* Now, it is fully initialized, so remove this flag here.
|
|
*/
|
|
clear_vm_uninitialized_flag(area);
|
|
|
|
kmemleak_vmalloc(area, size, gfp_mask);
|
|
|
|
return addr;
|
|
|
|
fail:
|
|
warn_alloc(gfp_mask, NULL,
|
|
"vmalloc: allocation failure: %lu bytes", real_size);
|
|
return NULL;
|
|
}
|
|
|
|
/**
|
|
* __vmalloc_node - allocate virtually contiguous memory
|
|
* @size: allocation size
|
|
* @align: desired alignment
|
|
* @gfp_mask: flags for the page level allocator
|
|
* @prot: protection mask for the allocated pages
|
|
* @node: node to use for allocation or NUMA_NO_NODE
|
|
* @caller: caller's return address
|
|
*
|
|
* Allocate enough pages to cover @size from the page level
|
|
* allocator with @gfp_mask flags. Map them into contiguous
|
|
* kernel virtual space, using a pagetable protection of @prot.
|
|
*
|
|
* Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL
|
|
* and __GFP_NOFAIL are not supported
|
|
*
|
|
* Any use of gfp flags outside of GFP_KERNEL should be consulted
|
|
* with mm people.
|
|
*
|
|
*/
|
|
static void *__vmalloc_node(unsigned long size, unsigned long align,
|
|
gfp_t gfp_mask, pgprot_t prot,
|
|
int node, const void *caller)
|
|
{
|
|
#ifdef CONFIG_ENABLE_VMALLOC_SAVING
|
|
return __vmalloc_node_range(size, align, PAGE_OFFSET, VMALLOC_END,
|
|
gfp_mask, prot, 0, node, caller);
|
|
#else
|
|
return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
|
|
gfp_mask, prot, 0, node, caller);
|
|
#endif
|
|
}
|
|
|
|
void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
|
|
{
|
|
return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE,
|
|
__builtin_return_address(0));
|
|
}
|
|
EXPORT_SYMBOL(__vmalloc);
|
|
|
|
static inline void *__vmalloc_node_flags(unsigned long size,
|
|
int node, gfp_t flags)
|
|
{
|
|
return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
|
|
node, __builtin_return_address(0));
|
|
}
|
|
|
|
|
|
void *__vmalloc_node_flags_caller(unsigned long size, int node, gfp_t flags,
|
|
void *caller)
|
|
{
|
|
return __vmalloc_node(size, 1, flags, PAGE_KERNEL, node, caller);
|
|
}
|
|
|
|
/**
|
|
* vmalloc - allocate virtually contiguous memory
|
|
* @size: allocation size
|
|
* Allocate enough pages to cover @size from the page level
|
|
* allocator and map them into contiguous kernel virtual space.
|
|
*
|
|
* For tight control over page level allocator and protection flags
|
|
* use __vmalloc() instead.
|
|
*/
|
|
void *vmalloc(unsigned long size)
|
|
{
|
|
return __vmalloc_node_flags(size, NUMA_NO_NODE,
|
|
GFP_KERNEL);
|
|
}
|
|
EXPORT_SYMBOL(vmalloc);
|
|
|
|
/**
|
|
* vzalloc - allocate virtually contiguous memory with zero fill
|
|
* @size: allocation size
|
|
* Allocate enough pages to cover @size from the page level
|
|
* allocator and map them into contiguous kernel virtual space.
|
|
* The memory allocated is set to zero.
|
|
*
|
|
* For tight control over page level allocator and protection flags
|
|
* use __vmalloc() instead.
|
|
*/
|
|
void *vzalloc(unsigned long size)
|
|
{
|
|
return __vmalloc_node_flags(size, NUMA_NO_NODE,
|
|
GFP_KERNEL | __GFP_ZERO);
|
|
}
|
|
EXPORT_SYMBOL(vzalloc);
|
|
|
|
/**
|
|
* vmalloc_user - allocate zeroed virtually contiguous memory for userspace
|
|
* @size: allocation size
|
|
*
|
|
* The resulting memory area is zeroed so it can be mapped to userspace
|
|
* without leaking data.
|
|
*/
|
|
void *vmalloc_user(unsigned long size)
|
|
{
|
|
struct vm_struct *area;
|
|
void *ret;
|
|
|
|
ret = __vmalloc_node(size, SHMLBA,
|
|
GFP_KERNEL | __GFP_ZERO,
|
|
PAGE_KERNEL, NUMA_NO_NODE,
|
|
__builtin_return_address(0));
|
|
if (ret) {
|
|
area = find_vm_area(ret);
|
|
area->flags |= VM_USERMAP;
|
|
}
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(vmalloc_user);
|
|
|
|
/**
|
|
* vmalloc_node - allocate memory on a specific node
|
|
* @size: allocation size
|
|
* @node: numa node
|
|
*
|
|
* Allocate enough pages to cover @size from the page level
|
|
* allocator and map them into contiguous kernel virtual space.
|
|
*
|
|
* For tight control over page level allocator and protection flags
|
|
* use __vmalloc() instead.
|
|
*/
|
|
void *vmalloc_node(unsigned long size, int node)
|
|
{
|
|
return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL,
|
|
node, __builtin_return_address(0));
|
|
}
|
|
EXPORT_SYMBOL(vmalloc_node);
|
|
|
|
/**
|
|
* vzalloc_node - allocate memory on a specific node with zero fill
|
|
* @size: allocation size
|
|
* @node: numa node
|
|
*
|
|
* Allocate enough pages to cover @size from the page level
|
|
* allocator and map them into contiguous kernel virtual space.
|
|
* The memory allocated is set to zero.
|
|
*
|
|
* For tight control over page level allocator and protection flags
|
|
* use __vmalloc_node() instead.
|
|
*/
|
|
void *vzalloc_node(unsigned long size, int node)
|
|
{
|
|
return __vmalloc_node_flags(size, node,
|
|
GFP_KERNEL | __GFP_ZERO);
|
|
}
|
|
EXPORT_SYMBOL(vzalloc_node);
|
|
|
|
/**
|
|
* vmalloc_exec - allocate virtually contiguous, executable memory
|
|
* @size: allocation size
|
|
*
|
|
* Kernel-internal function to allocate enough pages to cover @size
|
|
* the page level allocator and map them into contiguous and
|
|
* executable kernel virtual space.
|
|
*
|
|
* For tight control over page level allocator and protection flags
|
|
* use __vmalloc() instead.
|
|
*/
|
|
|
|
void *vmalloc_exec(unsigned long size)
|
|
{
|
|
return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL_EXEC,
|
|
NUMA_NO_NODE, __builtin_return_address(0));
|
|
}
|
|
|
|
#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
|
|
#define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL)
|
|
#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
|
|
#define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL)
|
|
#else
|
|
/*
|
|
* 64b systems should always have either DMA or DMA32 zones. For others
|
|
* GFP_DMA32 should do the right thing and use the normal zone.
|
|
*/
|
|
#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
|
|
#endif
|
|
|
|
/**
|
|
* vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
|
|
* @size: allocation size
|
|
*
|
|
* Allocate enough 32bit PA addressable pages to cover @size from the
|
|
* page level allocator and map them into contiguous kernel virtual space.
|
|
*/
|
|
void *vmalloc_32(unsigned long size)
|
|
{
|
|
return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
|
|
NUMA_NO_NODE, __builtin_return_address(0));
|
|
}
|
|
EXPORT_SYMBOL(vmalloc_32);
|
|
|
|
/**
|
|
* vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
|
|
* @size: allocation size
|
|
*
|
|
* The resulting memory area is 32bit addressable and zeroed so it can be
|
|
* mapped to userspace without leaking data.
|
|
*/
|
|
void *vmalloc_32_user(unsigned long size)
|
|
{
|
|
struct vm_struct *area;
|
|
void *ret;
|
|
|
|
ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
|
|
NUMA_NO_NODE, __builtin_return_address(0));
|
|
if (ret) {
|
|
area = find_vm_area(ret);
|
|
area->flags |= VM_USERMAP;
|
|
}
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(vmalloc_32_user);
|
|
|
|
/*
|
|
* small helper routine , copy contents to buf from addr.
|
|
* If the page is not present, fill zero.
|
|
*/
|
|
|
|
static int aligned_vread(char *buf, char *addr, unsigned long count)
|
|
{
|
|
struct page *p;
|
|
int copied = 0;
|
|
|
|
while (count) {
|
|
unsigned long offset, length;
|
|
|
|
offset = offset_in_page(addr);
|
|
length = PAGE_SIZE - offset;
|
|
if (length > count)
|
|
length = count;
|
|
p = vmalloc_to_page(addr);
|
|
/*
|
|
* To do safe access to this _mapped_ area, we need
|
|
* lock. But adding lock here means that we need to add
|
|
* overhead of vmalloc()/vfree() calles for this _debug_
|
|
* interface, rarely used. Instead of that, we'll use
|
|
* kmap() and get small overhead in this access function.
|
|
*/
|
|
if (p) {
|
|
/*
|
|
* we can expect USER0 is not used (see vread/vwrite's
|
|
* function description)
|
|
*/
|
|
void *map = kmap_atomic(p);
|
|
memcpy(buf, map + offset, length);
|
|
kunmap_atomic(map);
|
|
} else
|
|
memset(buf, 0, length);
|
|
|
|
addr += length;
|
|
buf += length;
|
|
copied += length;
|
|
count -= length;
|
|
}
|
|
return copied;
|
|
}
|
|
|
|
static int aligned_vwrite(char *buf, char *addr, unsigned long count)
|
|
{
|
|
struct page *p;
|
|
int copied = 0;
|
|
|
|
while (count) {
|
|
unsigned long offset, length;
|
|
|
|
offset = offset_in_page(addr);
|
|
length = PAGE_SIZE - offset;
|
|
if (length > count)
|
|
length = count;
|
|
p = vmalloc_to_page(addr);
|
|
/*
|
|
* To do safe access to this _mapped_ area, we need
|
|
* lock. But adding lock here means that we need to add
|
|
* overhead of vmalloc()/vfree() calles for this _debug_
|
|
* interface, rarely used. Instead of that, we'll use
|
|
* kmap() and get small overhead in this access function.
|
|
*/
|
|
if (p) {
|
|
/*
|
|
* we can expect USER0 is not used (see vread/vwrite's
|
|
* function description)
|
|
*/
|
|
void *map = kmap_atomic(p);
|
|
memcpy(map + offset, buf, length);
|
|
kunmap_atomic(map);
|
|
}
|
|
addr += length;
|
|
buf += length;
|
|
copied += length;
|
|
count -= length;
|
|
}
|
|
return copied;
|
|
}
|
|
|
|
/**
|
|
* vread() - read vmalloc area in a safe way.
|
|
* @buf: buffer for reading data
|
|
* @addr: vm address.
|
|
* @count: number of bytes to be read.
|
|
*
|
|
* Returns # of bytes which addr and buf should be increased.
|
|
* (same number to @count). Returns 0 if [addr...addr+count) doesn't
|
|
* includes any intersect with alive vmalloc area.
|
|
*
|
|
* This function checks that addr is a valid vmalloc'ed area, and
|
|
* copy data from that area to a given buffer. If the given memory range
|
|
* of [addr...addr+count) includes some valid address, data is copied to
|
|
* proper area of @buf. If there are memory holes, they'll be zero-filled.
|
|
* IOREMAP area is treated as memory hole and no copy is done.
|
|
*
|
|
* If [addr...addr+count) doesn't includes any intersects with alive
|
|
* vm_struct area, returns 0. @buf should be kernel's buffer.
|
|
*
|
|
* Note: In usual ops, vread() is never necessary because the caller
|
|
* should know vmalloc() area is valid and can use memcpy().
|
|
* This is for routines which have to access vmalloc area without
|
|
* any informaion, as /dev/kmem.
|
|
*
|
|
*/
|
|
|
|
long vread(char *buf, char *addr, unsigned long count)
|
|
{
|
|
struct vmap_area *va;
|
|
struct vm_struct *vm;
|
|
char *vaddr, *buf_start = buf;
|
|
unsigned long buflen = count;
|
|
unsigned long n;
|
|
|
|
/* Don't allow overflow */
|
|
if ((unsigned long) addr + count < count)
|
|
count = -(unsigned long) addr;
|
|
|
|
spin_lock(&vmap_area_lock);
|
|
list_for_each_entry(va, &vmap_area_list, list) {
|
|
if (!count)
|
|
break;
|
|
|
|
if (!(va->flags & VM_VM_AREA))
|
|
continue;
|
|
|
|
vm = va->vm;
|
|
vaddr = (char *) vm->addr;
|
|
if (addr >= vaddr + get_vm_area_size(vm))
|
|
continue;
|
|
while (addr < vaddr) {
|
|
if (count == 0)
|
|
goto finished;
|
|
*buf = '\0';
|
|
buf++;
|
|
addr++;
|
|
count--;
|
|
}
|
|
n = vaddr + get_vm_area_size(vm) - addr;
|
|
if (n > count)
|
|
n = count;
|
|
if (!(vm->flags & VM_IOREMAP))
|
|
aligned_vread(buf, addr, n);
|
|
else /* IOREMAP area is treated as memory hole */
|
|
memset(buf, 0, n);
|
|
buf += n;
|
|
addr += n;
|
|
count -= n;
|
|
}
|
|
finished:
|
|
spin_unlock(&vmap_area_lock);
|
|
|
|
if (buf == buf_start)
|
|
return 0;
|
|
/* zero-fill memory holes */
|
|
if (buf != buf_start + buflen)
|
|
memset(buf, 0, buflen - (buf - buf_start));
|
|
|
|
return buflen;
|
|
}
|
|
|
|
/**
|
|
* vwrite() - write vmalloc area in a safe way.
|
|
* @buf: buffer for source data
|
|
* @addr: vm address.
|
|
* @count: number of bytes to be read.
|
|
*
|
|
* Returns # of bytes which addr and buf should be incresed.
|
|
* (same number to @count).
|
|
* If [addr...addr+count) doesn't includes any intersect with valid
|
|
* vmalloc area, returns 0.
|
|
*
|
|
* This function checks that addr is a valid vmalloc'ed area, and
|
|
* copy data from a buffer to the given addr. If specified range of
|
|
* [addr...addr+count) includes some valid address, data is copied from
|
|
* proper area of @buf. If there are memory holes, no copy to hole.
|
|
* IOREMAP area is treated as memory hole and no copy is done.
|
|
*
|
|
* If [addr...addr+count) doesn't includes any intersects with alive
|
|
* vm_struct area, returns 0. @buf should be kernel's buffer.
|
|
*
|
|
* Note: In usual ops, vwrite() is never necessary because the caller
|
|
* should know vmalloc() area is valid and can use memcpy().
|
|
* This is for routines which have to access vmalloc area without
|
|
* any informaion, as /dev/kmem.
|
|
*/
|
|
|
|
long vwrite(char *buf, char *addr, unsigned long count)
|
|
{
|
|
struct vmap_area *va;
|
|
struct vm_struct *vm;
|
|
char *vaddr;
|
|
unsigned long n, buflen;
|
|
int copied = 0;
|
|
|
|
/* Don't allow overflow */
|
|
if ((unsigned long) addr + count < count)
|
|
count = -(unsigned long) addr;
|
|
buflen = count;
|
|
|
|
spin_lock(&vmap_area_lock);
|
|
list_for_each_entry(va, &vmap_area_list, list) {
|
|
if (!count)
|
|
break;
|
|
|
|
if (!(va->flags & VM_VM_AREA))
|
|
continue;
|
|
|
|
vm = va->vm;
|
|
vaddr = (char *) vm->addr;
|
|
if (addr >= vaddr + get_vm_area_size(vm))
|
|
continue;
|
|
while (addr < vaddr) {
|
|
if (count == 0)
|
|
goto finished;
|
|
buf++;
|
|
addr++;
|
|
count--;
|
|
}
|
|
n = vaddr + get_vm_area_size(vm) - addr;
|
|
if (n > count)
|
|
n = count;
|
|
if (!(vm->flags & VM_IOREMAP)) {
|
|
aligned_vwrite(buf, addr, n);
|
|
copied++;
|
|
}
|
|
buf += n;
|
|
addr += n;
|
|
count -= n;
|
|
}
|
|
finished:
|
|
spin_unlock(&vmap_area_lock);
|
|
if (!copied)
|
|
return 0;
|
|
return buflen;
|
|
}
|
|
|
|
/**
|
|
* remap_vmalloc_range_partial - map vmalloc pages to userspace
|
|
* @vma: vma to cover
|
|
* @uaddr: target user address to start at
|
|
* @kaddr: virtual address of vmalloc kernel memory
|
|
* @pgoff: offset from @kaddr to start at
|
|
* @size: size of map area
|
|
*
|
|
* Returns: 0 for success, -Exxx on failure
|
|
*
|
|
* This function checks that @kaddr is a valid vmalloc'ed area,
|
|
* and that it is big enough to cover the range starting at
|
|
* @uaddr in @vma. Will return failure if that criteria isn't
|
|
* met.
|
|
*
|
|
* Similar to remap_pfn_range() (see mm/memory.c)
|
|
*/
|
|
int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
|
|
void *kaddr, unsigned long pgoff,
|
|
unsigned long size)
|
|
{
|
|
struct vm_struct *area;
|
|
unsigned long off;
|
|
unsigned long end_index;
|
|
|
|
if (check_shl_overflow(pgoff, PAGE_SHIFT, &off))
|
|
return -EINVAL;
|
|
|
|
size = PAGE_ALIGN(size);
|
|
|
|
if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
|
|
return -EINVAL;
|
|
|
|
area = find_vm_area(kaddr);
|
|
if (!area)
|
|
return -EINVAL;
|
|
|
|
if (!(area->flags & VM_USERMAP))
|
|
return -EINVAL;
|
|
|
|
if (check_add_overflow(size, off, &end_index) ||
|
|
end_index > get_vm_area_size(area))
|
|
return -EINVAL;
|
|
kaddr += off;
|
|
|
|
do {
|
|
struct page *page = vmalloc_to_page(kaddr);
|
|
int ret;
|
|
|
|
ret = vm_insert_page(vma, uaddr, page);
|
|
if (ret)
|
|
return ret;
|
|
|
|
uaddr += PAGE_SIZE;
|
|
kaddr += PAGE_SIZE;
|
|
size -= PAGE_SIZE;
|
|
} while (size > 0);
|
|
|
|
vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(remap_vmalloc_range_partial);
|
|
|
|
/**
|
|
* remap_vmalloc_range - map vmalloc pages to userspace
|
|
* @vma: vma to cover (map full range of vma)
|
|
* @addr: vmalloc memory
|
|
* @pgoff: number of pages into addr before first page to map
|
|
*
|
|
* Returns: 0 for success, -Exxx on failure
|
|
*
|
|
* This function checks that addr is a valid vmalloc'ed area, and
|
|
* that it is big enough to cover the vma. Will return failure if
|
|
* that criteria isn't met.
|
|
*
|
|
* Similar to remap_pfn_range() (see mm/memory.c)
|
|
*/
|
|
int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
|
|
unsigned long pgoff)
|
|
{
|
|
return remap_vmalloc_range_partial(vma, vma->vm_start,
|
|
addr, pgoff,
|
|
vma->vm_end - vma->vm_start);
|
|
}
|
|
EXPORT_SYMBOL(remap_vmalloc_range);
|
|
|
|
/*
|
|
* Implement stubs for vmalloc_sync_[un]mappings () if the architecture chose
|
|
* not to have one.
|
|
*
|
|
* The purpose of this function is to make sure the vmalloc area
|
|
* mappings are identical in all page-tables in the system.
|
|
*/
|
|
void __weak vmalloc_sync_mappings(void)
|
|
{
|
|
}
|
|
|
|
void __weak vmalloc_sync_unmappings(void)
|
|
{
|
|
}
|
|
|
|
static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
|
|
{
|
|
pte_t ***p = data;
|
|
|
|
if (p) {
|
|
*(*p) = pte;
|
|
(*p)++;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* alloc_vm_area - allocate a range of kernel address space
|
|
* @size: size of the area
|
|
* @ptes: returns the PTEs for the address space
|
|
*
|
|
* Returns: NULL on failure, vm_struct on success
|
|
*
|
|
* This function reserves a range of kernel address space, and
|
|
* allocates pagetables to map that range. No actual mappings
|
|
* are created.
|
|
*
|
|
* If @ptes is non-NULL, pointers to the PTEs (in init_mm)
|
|
* allocated for the VM area are returned.
|
|
*/
|
|
struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
|
|
{
|
|
struct vm_struct *area;
|
|
|
|
area = get_vm_area_caller(size, VM_IOREMAP,
|
|
__builtin_return_address(0));
|
|
if (area == NULL)
|
|
return NULL;
|
|
|
|
/*
|
|
* This ensures that page tables are constructed for this region
|
|
* of kernel virtual address space and mapped into init_mm.
|
|
*/
|
|
if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
|
|
size, f, ptes ? &ptes : NULL)) {
|
|
free_vm_area(area);
|
|
return NULL;
|
|
}
|
|
|
|
return area;
|
|
}
|
|
EXPORT_SYMBOL_GPL(alloc_vm_area);
|
|
|
|
void free_vm_area(struct vm_struct *area)
|
|
{
|
|
struct vm_struct *ret;
|
|
ret = remove_vm_area(area->addr);
|
|
BUG_ON(ret != area);
|
|
kfree(area);
|
|
}
|
|
EXPORT_SYMBOL_GPL(free_vm_area);
|
|
|
|
#ifdef CONFIG_SMP
|
|
static struct vmap_area *node_to_va(struct rb_node *n)
|
|
{
|
|
return rb_entry_safe(n, struct vmap_area, rb_node);
|
|
}
|
|
|
|
/**
|
|
* pvm_find_va_enclose_addr - find the vmap_area @addr belongs to
|
|
* @addr: target address
|
|
*
|
|
* Returns: vmap_area if it is found. If there is no such area
|
|
* the first highest(reverse order) vmap_area is returned
|
|
* i.e. va->va_start < addr && va->va_end < addr or NULL
|
|
* if there are no any areas before @addr.
|
|
*/
|
|
static struct vmap_area *
|
|
pvm_find_va_enclose_addr(unsigned long addr)
|
|
{
|
|
struct vmap_area *va, *tmp;
|
|
struct rb_node *n;
|
|
|
|
n = free_vmap_area_root.rb_node;
|
|
va = NULL;
|
|
|
|
while (n) {
|
|
tmp = rb_entry(n, struct vmap_area, rb_node);
|
|
if (tmp->va_start <= addr) {
|
|
va = tmp;
|
|
if (tmp->va_end >= addr)
|
|
break;
|
|
|
|
n = n->rb_right;
|
|
} else {
|
|
n = n->rb_left;
|
|
}
|
|
}
|
|
|
|
return va;
|
|
}
|
|
|
|
/**
|
|
* pvm_determine_end_from_reverse - find the highest aligned address
|
|
* of free block below VMALLOC_END
|
|
* @va:
|
|
* in - the VA we start the search(reverse order);
|
|
* out - the VA with the highest aligned end address.
|
|
*
|
|
* Returns: determined end address within vmap_area
|
|
*/
|
|
static unsigned long
|
|
pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)
|
|
{
|
|
unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
|
|
unsigned long addr;
|
|
|
|
if (likely(*va)) {
|
|
list_for_each_entry_from_reverse((*va),
|
|
&free_vmap_area_list, list) {
|
|
addr = min((*va)->va_end & ~(align - 1), vmalloc_end);
|
|
if ((*va)->va_start < addr)
|
|
return addr;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
|
|
* @offsets: array containing offset of each area
|
|
* @sizes: array containing size of each area
|
|
* @nr_vms: the number of areas to allocate
|
|
* @align: alignment, all entries in @offsets and @sizes must be aligned to this
|
|
*
|
|
* Returns: kmalloc'd vm_struct pointer array pointing to allocated
|
|
* vm_structs on success, %NULL on failure
|
|
*
|
|
* Percpu allocator wants to use congruent vm areas so that it can
|
|
* maintain the offsets among percpu areas. This function allocates
|
|
* congruent vmalloc areas for it with GFP_KERNEL. These areas tend to
|
|
* be scattered pretty far, distance between two areas easily going up
|
|
* to gigabytes. To avoid interacting with regular vmallocs, these
|
|
* areas are allocated from top.
|
|
*
|
|
* Despite its complicated look, this allocator is rather simple. It
|
|
* does everything top-down and scans free blocks from the end looking
|
|
* for matching base. While scanning, if any of the areas do not fit the
|
|
* base address is pulled down to fit the area. Scanning is repeated till
|
|
* all the areas fit and then all necessary data structures are inserted
|
|
* and the result is returned.
|
|
*/
|
|
struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
|
|
const size_t *sizes, int nr_vms,
|
|
size_t align)
|
|
{
|
|
const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
|
|
const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
|
|
struct vmap_area **vas, *va;
|
|
struct vm_struct **vms;
|
|
int area, area2, last_area, term_area;
|
|
unsigned long base, start, size, end, last_end;
|
|
bool purged = false;
|
|
enum fit_type type;
|
|
|
|
/* verify parameters and allocate data structures */
|
|
BUG_ON(offset_in_page(align) || !is_power_of_2(align));
|
|
for (last_area = 0, area = 0; area < nr_vms; area++) {
|
|
start = offsets[area];
|
|
end = start + sizes[area];
|
|
|
|
/* is everything aligned properly? */
|
|
BUG_ON(!IS_ALIGNED(offsets[area], align));
|
|
BUG_ON(!IS_ALIGNED(sizes[area], align));
|
|
|
|
/* detect the area with the highest address */
|
|
if (start > offsets[last_area])
|
|
last_area = area;
|
|
|
|
for (area2 = area + 1; area2 < nr_vms; area2++) {
|
|
unsigned long start2 = offsets[area2];
|
|
unsigned long end2 = start2 + sizes[area2];
|
|
|
|
BUG_ON(start2 < end && start < end2);
|
|
}
|
|
}
|
|
last_end = offsets[last_area] + sizes[last_area];
|
|
|
|
if (vmalloc_end - vmalloc_start < last_end) {
|
|
WARN_ON(true);
|
|
return NULL;
|
|
}
|
|
|
|
vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
|
|
vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
|
|
if (!vas || !vms)
|
|
goto err_free2;
|
|
|
|
for (area = 0; area < nr_vms; area++) {
|
|
vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);
|
|
vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
|
|
if (!vas[area] || !vms[area])
|
|
goto err_free;
|
|
}
|
|
retry:
|
|
spin_lock(&vmap_area_lock);
|
|
|
|
/* start scanning - we scan from the top, begin with the last area */
|
|
area = term_area = last_area;
|
|
start = offsets[area];
|
|
end = start + sizes[area];
|
|
|
|
va = pvm_find_va_enclose_addr(vmalloc_end);
|
|
base = pvm_determine_end_from_reverse(&va, align) - end;
|
|
|
|
while (true) {
|
|
/*
|
|
* base might have underflowed, add last_end before
|
|
* comparing.
|
|
*/
|
|
if (base + last_end < vmalloc_start + last_end)
|
|
goto overflow;
|
|
|
|
/*
|
|
* Fitting base has not been found.
|
|
*/
|
|
if (va == NULL)
|
|
goto overflow;
|
|
|
|
/*
|
|
* If required width exeeds current VA block, move
|
|
* base downwards and then recheck.
|
|
*/
|
|
if (base + end > va->va_end) {
|
|
base = pvm_determine_end_from_reverse(&va, align) - end;
|
|
term_area = area;
|
|
continue;
|
|
}
|
|
|
|
/*
|
|
* If this VA does not fit, move base downwards and recheck.
|
|
*/
|
|
if (base + start < va->va_start) {
|
|
va = node_to_va(rb_prev(&va->rb_node));
|
|
base = pvm_determine_end_from_reverse(&va, align) - end;
|
|
term_area = area;
|
|
continue;
|
|
}
|
|
|
|
/*
|
|
* This area fits, move on to the previous one. If
|
|
* the previous one is the terminal one, we're done.
|
|
*/
|
|
area = (area + nr_vms - 1) % nr_vms;
|
|
if (area == term_area)
|
|
break;
|
|
|
|
start = offsets[area];
|
|
end = start + sizes[area];
|
|
va = pvm_find_va_enclose_addr(base + end);
|
|
}
|
|
|
|
/* we've found a fitting base, insert all va's */
|
|
for (area = 0; area < nr_vms; area++) {
|
|
int ret;
|
|
|
|
start = base + offsets[area];
|
|
size = sizes[area];
|
|
|
|
va = pvm_find_va_enclose_addr(start);
|
|
if (WARN_ON_ONCE(va == NULL))
|
|
/* It is a BUG(), but trigger recovery instead. */
|
|
goto recovery;
|
|
|
|
type = classify_va_fit_type(va, start, size);
|
|
if (WARN_ON_ONCE(type == NOTHING_FIT))
|
|
/* It is a BUG(), but trigger recovery instead. */
|
|
goto recovery;
|
|
|
|
ret = adjust_va_to_fit_type(va, start, size, type);
|
|
if (unlikely(ret))
|
|
goto recovery;
|
|
|
|
/* Allocated area. */
|
|
va = vas[area];
|
|
va->va_start = start;
|
|
va->va_end = start + size;
|
|
|
|
insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
|
|
}
|
|
|
|
spin_unlock(&vmap_area_lock);
|
|
|
|
/* insert all vm's */
|
|
for (area = 0; area < nr_vms; area++)
|
|
setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
|
|
pcpu_get_vm_areas);
|
|
|
|
kfree(vas);
|
|
return vms;
|
|
|
|
recovery:
|
|
/* Remove previously inserted areas. */
|
|
while (area--) {
|
|
__free_vmap_area(vas[area]);
|
|
vas[area] = NULL;
|
|
}
|
|
|
|
overflow:
|
|
spin_unlock(&vmap_area_lock);
|
|
if (!purged) {
|
|
purge_vmap_area_lazy();
|
|
purged = true;
|
|
|
|
/* Before "retry", check if we recover. */
|
|
for (area = 0; area < nr_vms; area++) {
|
|
if (vas[area])
|
|
continue;
|
|
|
|
vas[area] = kmem_cache_zalloc(
|
|
vmap_area_cachep, GFP_KERNEL);
|
|
if (!vas[area])
|
|
goto err_free;
|
|
}
|
|
|
|
goto retry;
|
|
}
|
|
|
|
err_free:
|
|
for (area = 0; area < nr_vms; area++) {
|
|
if (vas[area])
|
|
kmem_cache_free(vmap_area_cachep, vas[area]);
|
|
|
|
kfree(vms[area]);
|
|
}
|
|
err_free2:
|
|
kfree(vas);
|
|
kfree(vms);
|
|
return NULL;
|
|
}
|
|
|
|
/**
|
|
* pcpu_free_vm_areas - free vmalloc areas for percpu allocator
|
|
* @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
|
|
* @nr_vms: the number of allocated areas
|
|
*
|
|
* Free vm_structs and the array allocated by pcpu_get_vm_areas().
|
|
*/
|
|
void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < nr_vms; i++)
|
|
free_vm_area(vms[i]);
|
|
kfree(vms);
|
|
}
|
|
#endif /* CONFIG_SMP */
|
|
|
|
#ifdef CONFIG_PROC_FS
|
|
static void *s_start(struct seq_file *m, loff_t *pos)
|
|
__acquires(&vmap_area_lock)
|
|
{
|
|
spin_lock(&vmap_area_lock);
|
|
return seq_list_start(&vmap_area_list, *pos);
|
|
}
|
|
|
|
static void *s_next(struct seq_file *m, void *p, loff_t *pos)
|
|
{
|
|
return seq_list_next(p, &vmap_area_list, pos);
|
|
}
|
|
|
|
static void s_stop(struct seq_file *m, void *p)
|
|
__releases(&vmap_area_lock)
|
|
{
|
|
spin_unlock(&vmap_area_lock);
|
|
}
|
|
|
|
static void show_numa_info(struct seq_file *m, struct vm_struct *v)
|
|
{
|
|
if (IS_ENABLED(CONFIG_NUMA)) {
|
|
unsigned int nr, *counters = m->private;
|
|
|
|
if (!counters)
|
|
return;
|
|
|
|
if (v->flags & VM_UNINITIALIZED)
|
|
return;
|
|
/* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
|
|
smp_rmb();
|
|
|
|
memset(counters, 0, nr_node_ids * sizeof(unsigned int));
|
|
|
|
for (nr = 0; nr < v->nr_pages; nr++)
|
|
counters[page_to_nid(v->pages[nr])]++;
|
|
|
|
for_each_node_state(nr, N_HIGH_MEMORY)
|
|
if (counters[nr])
|
|
seq_printf(m, " N%u=%u", nr, counters[nr]);
|
|
}
|
|
}
|
|
|
|
static int s_show(struct seq_file *m, void *p)
|
|
{
|
|
struct vmap_area *va;
|
|
struct vm_struct *v;
|
|
|
|
va = list_entry(p, struct vmap_area, list);
|
|
|
|
/*
|
|
* s_show can encounter race with remove_vm_area, !VM_VM_AREA on
|
|
* behalf of vmap area is being tear down or vm_map_ram allocation.
|
|
*/
|
|
if (!(va->flags & VM_VM_AREA)) {
|
|
seq_printf(m, "0x%pK-0x%pK %7ld %s\n",
|
|
(void *)va->va_start, (void *)va->va_end,
|
|
va->va_end - va->va_start,
|
|
va->flags & VM_LAZY_FREE ? "unpurged vm_area" : "vm_map_ram");
|
|
|
|
return 0;
|
|
}
|
|
|
|
v = va->vm;
|
|
|
|
if (v->flags & VM_LOWMEM)
|
|
return 0;
|
|
|
|
seq_printf(m, "0x%pK-0x%pK %7ld",
|
|
v->addr, v->addr + v->size, v->size);
|
|
|
|
if (v->caller)
|
|
seq_printf(m, " %pS", v->caller);
|
|
|
|
if (v->nr_pages)
|
|
seq_printf(m, " pages=%d", v->nr_pages);
|
|
|
|
if (v->phys_addr)
|
|
seq_printf(m, " phys=%pa", &v->phys_addr);
|
|
|
|
if (v->flags & VM_IOREMAP)
|
|
seq_puts(m, " ioremap");
|
|
|
|
if (v->flags & VM_ALLOC)
|
|
seq_puts(m, " vmalloc");
|
|
|
|
if (v->flags & VM_MAP)
|
|
seq_puts(m, " vmap");
|
|
|
|
if (v->flags & VM_USERMAP)
|
|
seq_puts(m, " user");
|
|
|
|
if (is_vmalloc_addr(v->pages))
|
|
seq_puts(m, " vpages");
|
|
|
|
show_numa_info(m, v);
|
|
seq_putc(m, '\n');
|
|
return 0;
|
|
}
|
|
|
|
static const struct seq_operations vmalloc_op = {
|
|
.start = s_start,
|
|
.next = s_next,
|
|
.stop = s_stop,
|
|
.show = s_show,
|
|
};
|
|
|
|
static int __init proc_vmalloc_init(void)
|
|
{
|
|
if (IS_ENABLED(CONFIG_NUMA))
|
|
proc_create_seq_private("vmallocinfo", 0400, NULL,
|
|
&vmalloc_op,
|
|
nr_node_ids * sizeof(unsigned int), NULL);
|
|
else
|
|
proc_create_seq("vmallocinfo", 0400, NULL, &vmalloc_op);
|
|
return 0;
|
|
}
|
|
module_init(proc_vmalloc_init);
|
|
|
|
#endif
|
|
|