From 42311ff90dc8746bd81427b2ed6efda9af791b77 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 4 Aug 2010 12:07:08 +1000 Subject: drm/ttm: introduce utility function to free an allocated memory node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Existing core code/drivers call drm_mm_put_block on ttm_mem_reg.mm_node directly. Future patches will modify TTM behaviour in such a way that ttm_mem_reg.mm_node doesn't necessarily belong to drm_mm. Reviewed-by: Jerome Glisse Acked-by: Thomas Hellström Signed-off-by: Ben Skeggs --- drivers/gpu/drm/ttm/ttm_bo.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) (limited to 'drivers/gpu/drm/ttm/ttm_bo.c') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index cb4cf7ef4d1..80d37b460a8 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -475,11 +475,8 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all) list_del_init(&bo->ddestroy); ++put_count; } - if (bo->mem.mm_node) { - drm_mm_put_block(bo->mem.mm_node); - bo->mem.mm_node = NULL; - } spin_unlock(&glob->lru_lock); + ttm_bo_mem_put(bo, &bo->mem); atomic_set(&bo->reserved, 0); @@ -621,7 +618,6 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, bool no_wait_reserve, bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_global *glob = bo->glob; struct ttm_mem_reg evict_mem; struct ttm_placement placement; int ret = 0; @@ -667,12 +663,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, if (ret) { if (ret != -ERESTARTSYS) printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); - spin_lock(&glob->lru_lock); - if (evict_mem.mm_node) { - drm_mm_put_block(evict_mem.mm_node); - evict_mem.mm_node = NULL; - } - spin_unlock(&glob->lru_lock); + ttm_bo_mem_put(bo, &evict_mem); goto out; } bo->evicted = true; @@ -769,6 +760,19 @@ static int ttm_bo_man_get_node(struct ttm_buffer_object *bo, return 0; } +void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) +{ + struct ttm_bo_global *glob = bo->glob; + + if (mem->mm_node) { + spin_lock(&glob->lru_lock); + drm_mm_put_block(mem->mm_node); + spin_unlock(&glob->lru_lock); + mem->mm_node = NULL; + } +} +EXPORT_SYMBOL(ttm_bo_mem_put); + /** * Repeatedly evict memory from the LRU for @mem_type until we create enough * space, or we've evicted everything and there isn't enough space. -- cgit v1.2.3-70-g09d2 From d961db75ce86a84f1f04e91ad1014653ed7d9f46 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Thu, 5 Aug 2010 10:48:18 +1000 Subject: drm/ttm: restructure to allow driver to plug in alternate memory manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nouveau will need this on GeForce 8 and up to account for the GPU reordering physical VRAM for some memory types. Reviewed-by: Jerome Glisse Acked-by: Thomas Hellström Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bo.c | 18 ++-- drivers/gpu/drm/nouveau/nouveau_channel.c | 6 +- drivers/gpu/drm/nouveau/nouveau_notifier.c | 2 +- drivers/gpu/drm/nouveau/nouveau_sgdma.c | 4 +- drivers/gpu/drm/nouveau/nv50_crtc.c | 3 +- drivers/gpu/drm/nouveau/nv50_display.c | 2 +- drivers/gpu/drm/nouveau/nv50_instmem.c | 2 +- drivers/gpu/drm/nouveau/nvc0_instmem.c | 2 +- drivers/gpu/drm/radeon/radeon_object.c | 6 +- drivers/gpu/drm/radeon/radeon_ttm.c | 16 ++-- drivers/gpu/drm/ttm/Makefile | 3 +- drivers/gpu/drm/ttm/ttm_agp_backend.c | 3 +- drivers/gpu/drm/ttm/ttm_bo.c | 100 ++++--------------- drivers/gpu/drm/ttm/ttm_bo_manager.c | 148 +++++++++++++++++++++++++++++ drivers/gpu/drm/ttm/ttm_bo_util.c | 3 +- drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c | 3 +- include/drm/ttm/ttm_bo_api.h | 3 +- include/drm/ttm/ttm_bo_driver.h | 21 +++- 18 files changed, 229 insertions(+), 116 deletions(-) create mode 100644 drivers/gpu/drm/ttm/ttm_bo_manager.c (limited to 'drivers/gpu/drm/ttm/ttm_bo.c') diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index f685f392c22..80353e2b840 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -381,6 +381,7 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, man->default_caching = TTM_PL_FLAG_CACHED; break; case TTM_PL_VRAM: + man->func = &ttm_bo_manager_func; man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_FLAG_UNCACHED | @@ -392,6 +393,7 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, man->gpu_offset = 0; break; case TTM_PL_TT: + man->func = &ttm_bo_manager_func; switch (dev_priv->gart_info.type) { case NOUVEAU_GART_AGP: man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; @@ -494,8 +496,8 @@ nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, u64 src_offset, dst_offset; int ret; - src_offset = old_mem->mm_node->start << PAGE_SHIFT; - dst_offset = new_mem->mm_node->start << PAGE_SHIFT; + src_offset = old_mem->start << PAGE_SHIFT; + dst_offset = new_mem->start << PAGE_SHIFT; if (!nvbo->no_vm) { if (old_mem->mem_type == TTM_PL_VRAM) src_offset += dev_priv->vm_vram_base; @@ -597,8 +599,8 @@ static int nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) { - u32 src_offset = old_mem->mm_node->start << PAGE_SHIFT; - u32 dst_offset = new_mem->mm_node->start << PAGE_SHIFT; + u32 src_offset = old_mem->start << PAGE_SHIFT; + u32 dst_offset = new_mem->start << PAGE_SHIFT; u32 page_count = new_mem->num_pages; int ret; @@ -746,7 +748,7 @@ nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem, return 0; } - offset = new_mem->mm_node->start << PAGE_SHIFT; + offset = new_mem->start << PAGE_SHIFT; if (dev_priv->card_type == NV_50) { ret = nv50_mem_vm_bind_linear(dev, @@ -860,14 +862,14 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) case TTM_PL_TT: #if __OS_HAS_AGP if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) { - mem->bus.offset = mem->mm_node->start << PAGE_SHIFT; + mem->bus.offset = mem->start << PAGE_SHIFT; mem->bus.base = dev_priv->gart_info.aper_base; mem->bus.is_iomem = true; } #endif break; case TTM_PL_VRAM: - mem->bus.offset = mem->mm_node->start << PAGE_SHIFT; + mem->bus.offset = mem->start << PAGE_SHIFT; mem->bus.base = pci_resource_start(dev->pdev, 1); mem->bus.is_iomem = true; break; @@ -897,7 +899,7 @@ nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) } /* make sure bo is in mappable vram */ - if (bo->mem.mm_node->start + bo->mem.num_pages < dev_priv->fb_mappable_pages) + if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages) return 0; diff --git a/drivers/gpu/drm/nouveau/nouveau_channel.c b/drivers/gpu/drm/nouveau/nouveau_channel.c index 5eb4c966273..373950e3481 100644 --- a/drivers/gpu/drm/nouveau/nouveau_channel.c +++ b/drivers/gpu/drm/nouveau/nouveau_channel.c @@ -48,14 +48,14 @@ nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan) dev_priv->gart_info.aper_size, NV_DMA_ACCESS_RO, &pushbuf, NULL); - chan->pushbuf_base = pb->bo.mem.mm_node->start << PAGE_SHIFT; + chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT; } else if (dev_priv->card_type != NV_04) { ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0, dev_priv->fb_available_size, NV_DMA_ACCESS_RO, NV_DMA_TARGET_VIDMEM, &pushbuf); - chan->pushbuf_base = pb->bo.mem.mm_node->start << PAGE_SHIFT; + chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT; } else { /* NV04 cmdbuf hack, from original ddx.. not sure of it's * exact reason for existing :) PCI access to cmdbuf in @@ -67,7 +67,7 @@ nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan) dev_priv->fb_available_size, NV_DMA_ACCESS_RO, NV_DMA_TARGET_PCI, &pushbuf); - chan->pushbuf_base = pb->bo.mem.mm_node->start << PAGE_SHIFT; + chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT; } nouveau_gpuobj_ref(pushbuf, &chan->pushbuf); diff --git a/drivers/gpu/drm/nouveau/nouveau_notifier.c b/drivers/gpu/drm/nouveau/nouveau_notifier.c index 22b86189b7b..2cc59f8c658 100644 --- a/drivers/gpu/drm/nouveau/nouveau_notifier.c +++ b/drivers/gpu/drm/nouveau/nouveau_notifier.c @@ -113,7 +113,7 @@ nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle, return -ENOMEM; } - offset = chan->notifier_bo->bo.mem.mm_node->start << PAGE_SHIFT; + offset = chan->notifier_bo->bo.mem.start << PAGE_SHIFT; if (chan->notifier_bo->bo.mem.mem_type == TTM_PL_VRAM) { target = NV_DMA_TARGET_VIDMEM; } else diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c index 7f028fee7a5..288bacac7e5 100644 --- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c +++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c @@ -95,9 +95,9 @@ nouveau_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem) struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma; unsigned i, j, pte; - NV_DEBUG(dev, "pg=0x%lx\n", mem->mm_node->start); + NV_DEBUG(dev, "pg=0x%lx\n", mem->start); - pte = nouveau_sgdma_pte(nvbe->dev, mem->mm_node->start << PAGE_SHIFT); + pte = nouveau_sgdma_pte(nvbe->dev, mem->start << PAGE_SHIFT); nvbe->pte_start = pte; for (i = 0; i < nvbe->nr_pages; i++) { dma_addr_t dma_offset = nvbe->pages[i]; diff --git a/drivers/gpu/drm/nouveau/nv50_crtc.c b/drivers/gpu/drm/nouveau/nv50_crtc.c index 1686f8291b6..3f2fb4ec63a 100644 --- a/drivers/gpu/drm/nouveau/nv50_crtc.c +++ b/drivers/gpu/drm/nouveau/nv50_crtc.c @@ -104,8 +104,7 @@ nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked) OUT_RING(evo, nv_crtc->lut.depth == 8 ? NV50_EVO_CRTC_CLUT_MODE_OFF : NV50_EVO_CRTC_CLUT_MODE_ON); - OUT_RING(evo, (nv_crtc->lut.nvbo->bo.mem.mm_node->start << - PAGE_SHIFT) >> 8); + OUT_RING(evo, (nv_crtc->lut.nvbo->bo.mem.start << PAGE_SHIFT) >> 8); if (dev_priv->chipset != 0x50) { BEGIN_RING(evo, 0, NV84_EVO_CRTC(index, CLUT_DMA), 1); OUT_RING(evo, NvEvoVRAM); diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c index 11d366ad403..55c9663ef2b 100644 --- a/drivers/gpu/drm/nouveau/nv50_display.c +++ b/drivers/gpu/drm/nouveau/nv50_display.c @@ -345,7 +345,7 @@ nv50_display_init(struct drm_device *dev) /* initialise fifo */ nv_wr32(dev, NV50_PDISPLAY_CHANNEL_DMA_CB(0), - ((evo->pushbuf_bo->bo.mem.mm_node->start << PAGE_SHIFT) >> 8) | + ((evo->pushbuf_bo->bo.mem.start << PAGE_SHIFT) >> 8) | NV50_PDISPLAY_CHANNEL_DMA_CB_LOCATION_VRAM | NV50_PDISPLAY_CHANNEL_DMA_CB_VALID); nv_wr32(dev, NV50_PDISPLAY_CHANNEL_UNK2(0), 0x00010000); diff --git a/drivers/gpu/drm/nouveau/nv50_instmem.c b/drivers/gpu/drm/nouveau/nv50_instmem.c index f5800f21a9d..a53fc974332 100644 --- a/drivers/gpu/drm/nouveau/nv50_instmem.c +++ b/drivers/gpu/drm/nouveau/nv50_instmem.c @@ -347,7 +347,7 @@ nv50_instmem_populate(struct drm_device *dev, struct nouveau_gpuobj *gpuobj, return ret; } - gpuobj->vinst = gpuobj->im_backing->bo.mem.mm_node->start << PAGE_SHIFT; + gpuobj->vinst = gpuobj->im_backing->bo.mem.start << PAGE_SHIFT; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvc0_instmem.c b/drivers/gpu/drm/nouveau/nvc0_instmem.c index 6a41d644e04..13a0f78a908 100644 --- a/drivers/gpu/drm/nouveau/nvc0_instmem.c +++ b/drivers/gpu/drm/nouveau/nvc0_instmem.c @@ -50,7 +50,7 @@ nvc0_instmem_populate(struct drm_device *dev, struct nouveau_gpuobj *gpuobj, return ret; } - gpuobj->vinst = gpuobj->im_backing->bo.mem.mm_node->start << PAGE_SHIFT; + gpuobj->vinst = gpuobj->im_backing->bo.mem.start << PAGE_SHIFT; return 0; } diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c index 0afd1e62347..c26106066ec 100644 --- a/drivers/gpu/drm/radeon/radeon_object.c +++ b/drivers/gpu/drm/radeon/radeon_object.c @@ -435,7 +435,7 @@ int radeon_bo_get_surface_reg(struct radeon_bo *bo) out: radeon_set_surface_reg(rdev, i, bo->tiling_flags, bo->pitch, - bo->tbo.mem.mm_node->start << PAGE_SHIFT, + bo->tbo.mem.start << PAGE_SHIFT, bo->tbo.num_pages << PAGE_SHIFT); return 0; } @@ -532,7 +532,7 @@ int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo) rdev = rbo->rdev; if (bo->mem.mem_type == TTM_PL_VRAM) { size = bo->mem.num_pages << PAGE_SHIFT; - offset = bo->mem.mm_node->start << PAGE_SHIFT; + offset = bo->mem.start << PAGE_SHIFT; if ((offset + size) > rdev->mc.visible_vram_size) { /* hurrah the memory is not visible ! */ radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM); @@ -540,7 +540,7 @@ int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo) r = ttm_bo_validate(bo, &rbo->placement, false, true, false); if (unlikely(r != 0)) return r; - offset = bo->mem.mm_node->start << PAGE_SHIFT; + offset = bo->mem.start << PAGE_SHIFT; /* this should not happen */ if ((offset + size) > rdev->mc.visible_vram_size) return -EINVAL; diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index cc19aba9bb7..0921910698d 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -152,6 +152,7 @@ static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, man->default_caching = TTM_PL_FLAG_CACHED; break; case TTM_PL_TT: + man->func = &ttm_bo_manager_func; man->gpu_offset = rdev->mc.gtt_start; man->available_caching = TTM_PL_MASK_CACHING; man->default_caching = TTM_PL_FLAG_CACHED; @@ -173,6 +174,7 @@ static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, break; case TTM_PL_VRAM: /* "On-card" video ram */ + man->func = &ttm_bo_manager_func; man->gpu_offset = rdev->mc.vram_start; man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE; @@ -246,8 +248,8 @@ static int radeon_move_blit(struct ttm_buffer_object *bo, if (unlikely(r)) { return r; } - old_start = old_mem->mm_node->start << PAGE_SHIFT; - new_start = new_mem->mm_node->start << PAGE_SHIFT; + old_start = old_mem->start << PAGE_SHIFT; + new_start = new_mem->start << PAGE_SHIFT; switch (old_mem->mem_type) { case TTM_PL_VRAM: @@ -435,14 +437,14 @@ static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_ #if __OS_HAS_AGP if (rdev->flags & RADEON_IS_AGP) { /* RADEON_IS_AGP is set only if AGP is active */ - mem->bus.offset = mem->mm_node->start << PAGE_SHIFT; + mem->bus.offset = mem->start << PAGE_SHIFT; mem->bus.base = rdev->mc.agp_base; mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture; } #endif break; case TTM_PL_VRAM: - mem->bus.offset = mem->mm_node->start << PAGE_SHIFT; + mem->bus.offset = mem->start << PAGE_SHIFT; /* check if it's visible */ if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size) return -EINVAL; @@ -685,7 +687,7 @@ static int radeon_ttm_backend_bind(struct ttm_backend *backend, int r; gtt = container_of(backend, struct radeon_ttm_backend, backend); - gtt->offset = bo_mem->mm_node->start << PAGE_SHIFT; + gtt->offset = bo_mem->start << PAGE_SHIFT; if (!gtt->num_pages) { WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", gtt->num_pages, bo_mem, backend); } @@ -784,9 +786,9 @@ static int radeon_ttm_debugfs_init(struct radeon_device *rdev) radeon_mem_types_list[i].show = &radeon_mm_dump_table; radeon_mem_types_list[i].driver_features = 0; if (i == 0) - radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_VRAM].manager; + radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_VRAM].priv; else - radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_TT].manager; + radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_TT].priv; } /* Add ttm page pool to debugfs */ diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index b256d4adfaf..f3cf6f02c99 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -4,6 +4,7 @@ ccflags-y := -Iinclude/drm ttm-y := ttm_agp_backend.o ttm_memory.o ttm_tt.o ttm_bo.o \ ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ - ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o + ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o \ + ttm_bo_manager.o obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 4bf69c40449..f999e36f30b 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -74,6 +74,7 @@ static int ttm_agp_bind(struct ttm_backend *backend, struct ttm_mem_reg *bo_mem) { struct ttm_agp_backend *agp_be = container_of(backend, struct ttm_agp_backend, backend); + struct drm_mm_node *node = bo_mem->mm_node; struct agp_memory *mem = agp_be->mem; int cached = (bo_mem->placement & TTM_PL_FLAG_CACHED); int ret; @@ -81,7 +82,7 @@ static int ttm_agp_bind(struct ttm_backend *backend, struct ttm_mem_reg *bo_mem) mem->is_flushed = 1; mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY; - ret = agp_bind_memory(mem, bo_mem->mm_node->start); + ret = agp_bind_memory(mem, node->start); if (ret) printk(KERN_ERR TTM_PFX "AGP Bind memory failed.\n"); diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 80d37b460a8..af7b57a47fb 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -84,11 +84,8 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) man->available_caching); printk(KERN_ERR TTM_PFX " default_caching: 0x%08X\n", man->default_caching); - if (mem_type != TTM_PL_SYSTEM) { - spin_lock(&bdev->glob->lru_lock); - drm_mm_debug_table(&man->manager, TTM_PFX); - spin_unlock(&bdev->glob->lru_lock); - } + if (mem_type != TTM_PL_SYSTEM) + (*man->func->debug)(man, TTM_PFX); } static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, @@ -421,7 +418,7 @@ moved: if (bo->mem.mm_node) { spin_lock(&bo->lock); - bo->offset = (bo->mem.mm_node->start << PAGE_SHIFT) + + bo->offset = (bo->mem.start << PAGE_SHIFT) + bdev->man[bo->mem.mem_type].gpu_offset; bo->cur_placement = bo->mem.placement; spin_unlock(&bo->lock); @@ -724,52 +721,12 @@ retry: return ret; } -static int ttm_bo_man_get_node(struct ttm_buffer_object *bo, - struct ttm_mem_type_manager *man, - struct ttm_placement *placement, - struct ttm_mem_reg *mem, - struct drm_mm_node **node) -{ - struct ttm_bo_global *glob = bo->glob; - unsigned long lpfn; - int ret; - - lpfn = placement->lpfn; - if (!lpfn) - lpfn = man->size; - *node = NULL; - do { - ret = drm_mm_pre_get(&man->manager); - if (unlikely(ret)) - return ret; - - spin_lock(&glob->lru_lock); - *node = drm_mm_search_free_in_range(&man->manager, - mem->num_pages, mem->page_alignment, - placement->fpfn, lpfn, 1); - if (unlikely(*node == NULL)) { - spin_unlock(&glob->lru_lock); - return 0; - } - *node = drm_mm_get_block_atomic_range(*node, mem->num_pages, - mem->page_alignment, - placement->fpfn, - lpfn); - spin_unlock(&glob->lru_lock); - } while (*node == NULL); - return 0; -} - void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) { - struct ttm_bo_global *glob = bo->glob; + struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; - if (mem->mm_node) { - spin_lock(&glob->lru_lock); - drm_mm_put_block(mem->mm_node); - spin_unlock(&glob->lru_lock); - mem->mm_node = NULL; - } + if (mem->mm_node) + (*man->func->put_node)(man, mem); } EXPORT_SYMBOL(ttm_bo_mem_put); @@ -788,14 +745,13 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_global *glob = bdev->glob; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - struct drm_mm_node *node; int ret; do { - ret = ttm_bo_man_get_node(bo, man, placement, mem, &node); + ret = (*man->func->get_node)(man, bo, placement, mem); if (unlikely(ret != 0)) return ret; - if (node) + if (mem->mm_node) break; spin_lock(&glob->lru_lock); if (list_empty(&man->lru)) { @@ -808,9 +764,8 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, if (unlikely(ret != 0)) return ret; } while (1); - if (node == NULL) + if (mem->mm_node == NULL) return -ENOMEM; - mem->mm_node = node; mem->mem_type = mem_type; return 0; } @@ -884,7 +839,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, bool type_found = false; bool type_ok = false; bool has_erestartsys = false; - struct drm_mm_node *node = NULL; int i, ret; mem->mm_node = NULL; @@ -918,17 +872,15 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, if (man->has_type && man->use_type) { type_found = true; - ret = ttm_bo_man_get_node(bo, man, placement, mem, - &node); + ret = (*man->func->get_node)(man, bo, placement, mem); if (unlikely(ret)) return ret; } - if (node) + if (mem->mm_node) break; } - if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || node) { - mem->mm_node = node; + if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { mem->mem_type = mem_type; mem->placement = cur_flags; return 0; @@ -998,7 +950,6 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo, bool interruptible, bool no_wait_reserve, bool no_wait_gpu) { - struct ttm_bo_global *glob = bo->glob; int ret = 0; struct ttm_mem_reg mem; @@ -1026,11 +977,8 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo, goto out_unlock; ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu); out_unlock: - if (ret && mem.mm_node) { - spin_lock(&glob->lru_lock); - drm_mm_put_block(mem.mm_node); - spin_unlock(&glob->lru_lock); - } + if (ret && mem.mm_node) + ttm_bo_mem_put(bo, &mem); return ret; } @@ -1038,11 +986,10 @@ static int ttm_bo_mem_compat(struct ttm_placement *placement, struct ttm_mem_reg *mem) { int i; - struct drm_mm_node *node = mem->mm_node; - if (node && placement->lpfn != 0 && - (node->start < placement->fpfn || - node->start + node->size > placement->lpfn)) + if (mem->mm_node && placement->lpfn != 0 && + (mem->start < placement->fpfn || + mem->start + mem->num_pages > placement->lpfn)) return -1; for (i = 0; i < placement->num_placement; i++) { @@ -1286,7 +1233,6 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) { - struct ttm_bo_global *glob = bdev->glob; struct ttm_mem_type_manager *man; int ret = -EINVAL; @@ -1309,13 +1255,7 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) if (mem_type > 0) { ttm_bo_force_list_clean(bdev, mem_type, false); - spin_lock(&glob->lru_lock); - if (drm_mm_clean(&man->manager)) - drm_mm_takedown(&man->manager); - else - ret = -EBUSY; - - spin_unlock(&glob->lru_lock); + ret = (*man->func->takedown)(man); } return ret; @@ -1366,6 +1306,7 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, ret = bdev->driver->init_mem_type(bdev, type, man); if (ret) return ret; + man->bdev = bdev; ret = 0; if (type != TTM_PL_SYSTEM) { @@ -1375,7 +1316,8 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, type); return ret; } - ret = drm_mm_init(&man->manager, 0, p_size); + + ret = (*man->func->init)(man, p_size); if (ret) return ret; } diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c new file mode 100644 index 00000000000..7410c190c89 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c @@ -0,0 +1,148 @@ +/************************************************************************** + * + * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#include "ttm/ttm_module.h" +#include "ttm/ttm_bo_driver.h" +#include "ttm/ttm_placement.h" +#include +#include +#include +#include +#include +#include + +static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, + struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_mem_reg *mem) +{ + struct ttm_bo_global *glob = man->bdev->glob; + struct drm_mm *mm = man->priv; + struct drm_mm_node *node = NULL; + unsigned long lpfn; + int ret; + + lpfn = placement->lpfn; + if (!lpfn) + lpfn = man->size; + do { + ret = drm_mm_pre_get(mm); + if (unlikely(ret)) + return ret; + + spin_lock(&glob->lru_lock); + node = drm_mm_search_free_in_range(mm, + mem->num_pages, mem->page_alignment, + placement->fpfn, lpfn, 1); + if (unlikely(node == NULL)) { + spin_unlock(&glob->lru_lock); + return 0; + } + node = drm_mm_get_block_atomic_range(node, mem->num_pages, + mem->page_alignment, + placement->fpfn, + lpfn); + spin_unlock(&glob->lru_lock); + } while (node == NULL); + + mem->mm_node = node; + mem->start = node->start; + return 0; +} + +static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man, + struct ttm_mem_reg *mem) +{ + struct ttm_bo_global *glob = man->bdev->glob; + + if (mem->mm_node) { + spin_lock(&glob->lru_lock); + drm_mm_put_block(mem->mm_node); + spin_unlock(&glob->lru_lock); + mem->mm_node = NULL; + } +} + +static int ttm_bo_man_init(struct ttm_mem_type_manager *man, + unsigned long p_size) +{ + struct drm_mm *mm; + int ret; + + mm = kzalloc(sizeof(*mm), GFP_KERNEL); + if (!mm) + return -ENOMEM; + + ret = drm_mm_init(mm, 0, p_size); + if (ret) { + kfree(mm); + return ret; + } + + man->priv = mm; + return 0; +} + +static int ttm_bo_man_takedown(struct ttm_mem_type_manager *man) +{ + struct ttm_bo_global *glob = man->bdev->glob; + struct drm_mm *mm = man->priv; + int ret = 0; + + spin_lock(&glob->lru_lock); + if (drm_mm_clean(mm)) { + drm_mm_takedown(mm); + kfree(mm); + man->priv = NULL; + } else + ret = -EBUSY; + spin_unlock(&glob->lru_lock); + return ret; +} + +static void ttm_bo_man_debug(struct ttm_mem_type_manager *man, + const char *prefix) +{ + struct ttm_bo_global *glob = man->bdev->glob; + struct drm_mm *mm = man->priv; + + spin_lock(&glob->lru_lock); + drm_mm_debug_table(mm, prefix); + spin_unlock(&glob->lru_lock); +} + +const struct ttm_mem_type_manager_func ttm_bo_manager_func = { + ttm_bo_man_init, + ttm_bo_man_takedown, + ttm_bo_man_get_node, + ttm_bo_man_put_node, + ttm_bo_man_debug +}; +EXPORT_SYMBOL(ttm_bo_manager_func); diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 0ebfe0d9493..c9d2d4d8d06 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -256,8 +256,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, dir = 1; if ((old_mem->mem_type == new_mem->mem_type) && - (new_mem->mm_node->start < - old_mem->mm_node->start + old_mem->mm_node->size)) { + (new_mem->start < old_mem->start + old_mem->size)) { dir = -1; add = new_mem->num_pages - 1; } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c index c4f5114aee7..1b3bd8c6c67 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c @@ -147,6 +147,7 @@ int vmw_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, break; case TTM_PL_VRAM: /* "On-card" video ram */ + man->func = &ttm_bo_manager_func; man->gpu_offset = 0; man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_MASK_CACHING; @@ -203,7 +204,7 @@ static int vmw_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg /* System memory */ return 0; case TTM_PL_VRAM: - mem->bus.offset = mem->mm_node->start << PAGE_SHIFT; + mem->bus.offset = mem->start << PAGE_SHIFT; mem->bus.base = dev_priv->vram_start; mem->bus.is_iomem = true; break; diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index 267a86c74e2..49b43c23636 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -102,7 +102,8 @@ struct ttm_bus_placement { */ struct ttm_mem_reg { - struct drm_mm_node *mm_node; + void *mm_node; + unsigned long start; unsigned long size; unsigned long num_pages; uint32_t page_alignment; diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 6c694d86e03..e3371dbe6a1 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -203,7 +203,22 @@ struct ttm_tt { * It's set up by the ttm_bo_driver::init_mem_type method. */ +struct ttm_mem_type_manager; + +struct ttm_mem_type_manager_func { + int (*init)(struct ttm_mem_type_manager *man, unsigned long p_size); + int (*takedown)(struct ttm_mem_type_manager *man); + int (*get_node)(struct ttm_mem_type_manager *man, + struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_mem_reg *mem); + void (*put_node)(struct ttm_mem_type_manager *man, + struct ttm_mem_reg *mem); + void (*debug)(struct ttm_mem_type_manager *man, const char *prefix); +}; + struct ttm_mem_type_manager { + struct ttm_bo_device *bdev; /* * No protection. Constant from start. @@ -222,8 +237,8 @@ struct ttm_mem_type_manager { * TODO: Consider one lru_lock per ttm_mem_type_manager. * Plays ill with list removal, though. */ - - struct drm_mm manager; + const struct ttm_mem_type_manager_func *func; + void *priv; struct list_head lru; }; @@ -895,6 +910,8 @@ extern int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, */ extern pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp); +extern const struct ttm_mem_type_manager_func ttm_bo_manager_func; + #if (defined(CONFIG_AGP) || (defined(CONFIG_AGP_MODULE) && defined(MODULE))) #define TTM_HAS_AGP #include -- cgit v1.2.3-70-g09d2 From c9220b0f7cbd1d2272426aa81a72ae2f6582bb71 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Fri, 8 Oct 2010 08:57:10 +1000 Subject: drm/ttm: add unlocked variant of new manager put node. We need the unlocked variant for the new codepath introduced to fix the race condition in master recently. Signed-off-by: Dave Airlie --- drivers/gpu/drm/ttm/ttm_bo.c | 13 ++++++++++--- drivers/gpu/drm/ttm/ttm_bo_manager.c | 10 ++++++++++ include/drm/ttm/ttm_bo_driver.h | 4 ++++ 3 files changed, 24 insertions(+), 3 deletions(-) (limited to 'drivers/gpu/drm/ttm/ttm_bo.c') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 1e9bb2156dc..5ef0103bd0b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -464,9 +464,7 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) spin_lock(&glob->lru_lock); } - if (bo->mem.mm_node) { - ttm_bo_mem_put(bo, &bo->mem); - } + ttm_bo_mem_put_locked(bo, &bo->mem); atomic_set(&bo->reserved, 0); wake_up_all(&bo->event_queue); @@ -791,6 +789,15 @@ void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) } EXPORT_SYMBOL(ttm_bo_mem_put); +void ttm_bo_mem_put_locked(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) +{ + struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; + + if (mem->mm_node) + (*man->func->put_node_locked)(man, mem); +} +EXPORT_SYMBOL(ttm_bo_mem_put_locked); + /** * Repeatedly evict memory from the LRU for @mem_type until we create enough * space, or we've evicted everything and there isn't enough space. diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c index 7410c190c89..35c97b20bda 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_manager.c +++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c @@ -90,6 +90,15 @@ static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man, } } +static void ttm_bo_man_put_node_locked(struct ttm_mem_type_manager *man, + struct ttm_mem_reg *mem) +{ + if (mem->mm_node) { + drm_mm_put_block(mem->mm_node); + mem->mm_node = NULL; + } +} + static int ttm_bo_man_init(struct ttm_mem_type_manager *man, unsigned long p_size) { @@ -143,6 +152,7 @@ const struct ttm_mem_type_manager_func ttm_bo_manager_func = { ttm_bo_man_takedown, ttm_bo_man_get_node, ttm_bo_man_put_node, + ttm_bo_man_put_node_locked, ttm_bo_man_debug }; EXPORT_SYMBOL(ttm_bo_manager_func); diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index e3371dbe6a1..d0ff529fedd 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -214,6 +214,8 @@ struct ttm_mem_type_manager_func { struct ttm_mem_reg *mem); void (*put_node)(struct ttm_mem_type_manager *man, struct ttm_mem_reg *mem); + void (*put_node_locked)(struct ttm_mem_type_manager *man, + struct ttm_mem_reg *mem); void (*debug)(struct ttm_mem_type_manager *man, const char *prefix); }; @@ -667,6 +669,8 @@ extern int ttm_bo_mem_space(struct ttm_buffer_object *bo, extern void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem); +extern void ttm_bo_mem_put_locked(struct ttm_buffer_object *bo, + struct ttm_mem_reg *mem); /** * ttm_bo_wait_for_cpu -- cgit v1.2.3-70-g09d2 From 965d38074e6eae71757a8baf9a348139e1e6894d Mon Sep 17 00:00:00 2001 From: Jean Delvare Date: Sat, 9 Oct 2010 12:36:45 +0000 Subject: drm/ttm: Simplify ttm_bo_wait_unreserved Function ttm_bo_wait_unreserved can be slightly simplified. Signed-off-by: Jean Delvare Cc: Thomas Hellstrom Cc: Jerome Glisse Signed-off-by: Dave Airlie --- drivers/gpu/drm/ttm/ttm_bo.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'drivers/gpu/drm/ttm/ttm_bo.c') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 5ef0103bd0b..9f2eed520fc 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -166,18 +166,13 @@ static void ttm_bo_release_list(struct kref *list_kref) int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) { - if (interruptible) { - int ret = 0; - - ret = wait_event_interruptible(bo->event_queue, + return wait_event_interruptible(bo->event_queue, atomic_read(&bo->reserved) == 0); - if (unlikely(ret != 0)) - return ret; } else { wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0); + return 0; } - return 0; } EXPORT_SYMBOL(ttm_bo_wait_unreserved); -- cgit v1.2.3-70-g09d2 From 40d857bba2915a4e8d82f44744a186bfdd1a46ea Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Tue, 19 Oct 2010 09:01:00 +0200 Subject: drm/ttm: Avoid using the ttm_mem_type_manager::put_locked function Release the lru spinlock early. Signed-off-by: Thomas Hellstrom Signed-off-by: Dave Airlie --- drivers/gpu/drm/ttm/ttm_bo.c | 32 ++++++-------------------------- drivers/gpu/drm/ttm/ttm_bo_manager.c | 10 ---------- include/drm/ttm/ttm_bo_driver.h | 2 -- 3 files changed, 6 insertions(+), 38 deletions(-) (limited to 'drivers/gpu/drm/ttm/ttm_bo.c') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 9f2eed520fc..4a73f401644 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -434,36 +434,25 @@ out_err: } /** - * Call bo::reserved and with the lru lock held. + * Call bo::reserved. * Will release GPU memory type usage on destruction. - * This is the place to put in driver specific hooks. - * Will release the bo::reserved lock and the - * lru lock on exit. + * This is the place to put in driver specific hooks to release + * driver private resources. + * Will release the bo::reserved lock. */ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) { - struct ttm_bo_global *glob = bo->glob; - if (bo->ttm) { - - /** - * Release the lru_lock, since we don't want to have - * an atomic requirement on ttm_tt[unbind|destroy]. - */ - - spin_unlock(&glob->lru_lock); ttm_tt_unbind(bo->ttm); ttm_tt_destroy(bo->ttm); bo->ttm = NULL; - spin_lock(&glob->lru_lock); } - ttm_bo_mem_put_locked(bo, &bo->mem); + ttm_bo_mem_put(bo, &bo->mem); atomic_set(&bo->reserved, 0); wake_up_all(&bo->event_queue); - spin_unlock(&glob->lru_lock); } @@ -528,7 +517,7 @@ retry: list_del_init(&bo->ddestroy); ++put_count; } - + spin_unlock(&glob->lru_lock); ttm_bo_cleanup_memtype_use(bo); while (put_count--) @@ -784,15 +773,6 @@ void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) } EXPORT_SYMBOL(ttm_bo_mem_put); -void ttm_bo_mem_put_locked(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) -{ - struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; - - if (mem->mm_node) - (*man->func->put_node_locked)(man, mem); -} -EXPORT_SYMBOL(ttm_bo_mem_put_locked); - /** * Repeatedly evict memory from the LRU for @mem_type until we create enough * space, or we've evicted everything and there isn't enough space. diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c index 35c97b20bda..7410c190c89 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_manager.c +++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c @@ -90,15 +90,6 @@ static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man, } } -static void ttm_bo_man_put_node_locked(struct ttm_mem_type_manager *man, - struct ttm_mem_reg *mem) -{ - if (mem->mm_node) { - drm_mm_put_block(mem->mm_node); - mem->mm_node = NULL; - } -} - static int ttm_bo_man_init(struct ttm_mem_type_manager *man, unsigned long p_size) { @@ -152,7 +143,6 @@ const struct ttm_mem_type_manager_func ttm_bo_manager_func = { ttm_bo_man_takedown, ttm_bo_man_get_node, ttm_bo_man_put_node, - ttm_bo_man_put_node_locked, ttm_bo_man_debug }; EXPORT_SYMBOL(ttm_bo_manager_func); diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index d0ff529fedd..d01b4ddbdc5 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -214,8 +214,6 @@ struct ttm_mem_type_manager_func { struct ttm_mem_reg *mem); void (*put_node)(struct ttm_mem_type_manager *man, struct ttm_mem_reg *mem); - void (*put_node_locked)(struct ttm_mem_type_manager *man, - struct ttm_mem_reg *mem); void (*debug)(struct ttm_mem_type_manager *man, const char *prefix); }; -- cgit v1.2.3-70-g09d2 From e1efc9b6ac22c605fd326b3f6af9b393325d43b4 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Tue, 19 Oct 2010 09:01:01 +0200 Subject: drm/ttm: Optimize delayed buffer destruction This commit replaces the ttm_bo_cleanup_ref function with two new functions. One for the case where the bo is not yet on the delayed destroy list, and one for the case where the bo was on the delayed destroy list, at least at the time of call. This makes it possible to optimize the two cases somewhat. It also enables the possibility to directly destroy buffers on the delayed delete list when they are about to be evicted or swapped out. Currently they were only evicted / swapped and destruction was left for the delayed buffer destruction thread. Signed-off-by: Thomas Hellstrom Signed-off-by: Dave Airlie --- drivers/gpu/drm/ttm/ttm_bo.c | 171 +++++++++++++++++++++++++++---------------- 1 file changed, 107 insertions(+), 64 deletions(-) (limited to 'drivers/gpu/drm/ttm/ttm_bo.c') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 4a73f401644..a1cb783c713 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -455,100 +455,123 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) wake_up_all(&bo->event_queue); } - -/** - * If bo idle, remove from delayed- and lru lists, and unref. - * If not idle, and already on delayed list, do nothing. - * If not idle, and not on delayed list, put on delayed list, - * up the list_kref and schedule a delayed list check. - */ - -static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all) +static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_global *glob = bo->glob; - struct ttm_bo_driver *driver = bdev->driver; + struct ttm_bo_driver *driver; + void *sync_obj; + void *sync_obj_arg; + int put_count; int ret; spin_lock(&bo->lock); -retry: - (void) ttm_bo_wait(bo, false, false, !remove_all); - + (void) ttm_bo_wait(bo, false, false, true); if (!bo->sync_obj) { - int put_count; - - spin_unlock(&bo->lock); spin_lock(&glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, false, !remove_all, false, 0); /** - * Someone else has the object reserved. Bail and retry. + * Lock inversion between bo::reserve and bo::lock here, + * but that's OK, since we're only trylocking. */ - if (unlikely(ret == -EBUSY)) { - spin_unlock(&glob->lru_lock); - spin_lock(&bo->lock); - goto requeue; - } - - /** - * We can re-check for sync object without taking - * the bo::lock since setting the sync object requires - * also bo::reserved. A busy object at this point may - * be caused by another thread starting an accelerated - * eviction. - */ + ret = ttm_bo_reserve_locked(bo, false, true, false, 0); - if (unlikely(bo->sync_obj)) { - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); - spin_unlock(&glob->lru_lock); - spin_lock(&bo->lock); - if (remove_all) - goto retry; - else - goto requeue; - } + if (unlikely(ret == -EBUSY)) + goto queue; + spin_unlock(&bo->lock); put_count = ttm_bo_del_from_lru(bo); - if (!list_empty(&bo->ddestroy)) { - list_del_init(&bo->ddestroy); - ++put_count; - } spin_unlock(&glob->lru_lock); ttm_bo_cleanup_memtype_use(bo); while (put_count--) kref_put(&bo->list_kref, ttm_bo_ref_bug); - return 0; + return; + } else { + spin_lock(&glob->lru_lock); } -requeue: +queue: + sync_obj = bo->sync_obj; + sync_obj_arg = bo->sync_obj_arg; + driver = bdev->driver; + + kref_get(&bo->list_kref); + list_add_tail(&bo->ddestroy, &bdev->ddestroy); + spin_unlock(&glob->lru_lock); + spin_unlock(&bo->lock); + + if (sync_obj) + driver->sync_obj_flush(sync_obj, sync_obj_arg); + schedule_delayed_work(&bdev->wq, + ((HZ / 100) < 1) ? 1 : HZ / 100); +} + +/** + * function ttm_bo_cleanup_refs + * If bo idle, remove from delayed- and lru lists, and unref. + * If not idle, do nothing. + * + * @interruptible Any sleeps should occur interruptibly. + * @no_wait_reserve Never wait for reserve. Return -EBUSY instead. + * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. + */ + +static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait_reserve, + bool no_wait_gpu) +{ + struct ttm_bo_global *glob = bo->glob; + int put_count; + int ret = 0; + +retry: + spin_lock(&bo->lock); + ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); + spin_unlock(&bo->lock); + + if (unlikely(ret != 0)) + return ret; + spin_lock(&glob->lru_lock); - if (list_empty(&bo->ddestroy)) { - void *sync_obj = bo->sync_obj; - void *sync_obj_arg = bo->sync_obj_arg; + ret = ttm_bo_reserve_locked(bo, interruptible, + no_wait_reserve, false, 0); - kref_get(&bo->list_kref); - list_add_tail(&bo->ddestroy, &bdev->ddestroy); + if (unlikely(ret != 0) || list_empty(&bo->ddestroy)) { spin_unlock(&glob->lru_lock); - spin_unlock(&bo->lock); + return ret; + } - if (sync_obj) - driver->sync_obj_flush(sync_obj, sync_obj_arg); - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); - ret = 0; + /** + * We can re-check for sync object without taking + * the bo::lock since setting the sync object requires + * also bo::reserved. A busy object at this point may + * be caused by another thread recently starting an accelerated + * eviction. + */ - } else { + if (unlikely(bo->sync_obj)) { + atomic_set(&bo->reserved, 0); + wake_up_all(&bo->event_queue); spin_unlock(&glob->lru_lock); - spin_unlock(&bo->lock); - ret = -EBUSY; + goto retry; } - return ret; + put_count = ttm_bo_del_from_lru(bo); + list_del_init(&bo->ddestroy); + ++put_count; + + spin_unlock(&glob->lru_lock); + ttm_bo_cleanup_memtype_use(bo); + + while (put_count--) + kref_put(&bo->list_kref, ttm_bo_ref_bug); + + return 0; } /** @@ -580,7 +603,8 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) } spin_unlock(&glob->lru_lock); - ret = ttm_bo_cleanup_refs(entry, remove_all); + ret = ttm_bo_cleanup_refs(entry, false, !remove_all, + !remove_all); kref_put(&entry->list_kref, ttm_bo_release_list); entry = nentry; @@ -623,7 +647,7 @@ static void ttm_bo_release(struct kref *kref) bo->vm_node = NULL; } write_unlock(&bdev->vm_lock); - ttm_bo_cleanup_refs(bo, false); + ttm_bo_cleanup_refs_or_queue(bo); kref_put(&bo->list_kref, ttm_bo_release_list); write_lock(&bdev->vm_lock); } @@ -731,6 +755,18 @@ retry: bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); kref_get(&bo->list_kref); + if (!list_empty(&bo->ddestroy)) { + spin_unlock(&glob->lru_lock); + ret = ttm_bo_cleanup_refs(bo, interruptible, + no_wait_reserve, no_wait_gpu); + kref_put(&bo->list_kref, ttm_bo_release_list); + + if (likely(ret == 0 || ret == -ERESTARTSYS)) + return ret; + + goto retry; + } + ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0); if (unlikely(ret == -EBUSY)) { @@ -1754,6 +1790,13 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) struct ttm_buffer_object, swap); kref_get(&bo->list_kref); + if (!list_empty(&bo->ddestroy)) { + spin_unlock(&glob->lru_lock); + (void) ttm_bo_cleanup_refs(bo, false, false, false); + kref_put(&bo->list_kref, ttm_bo_release_list); + continue; + } + /** * Reserve buffer. Since we unlock while sleeping, we need * to re-check that nobody removed us from the swap-list while -- cgit v1.2.3-70-g09d2