1/* 2 * Copyright (C) 2007 Ben Skeggs. 3 * 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining 7 * a copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sublicense, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial 16 * portions of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 21 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 22 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 24 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 */ 27 28#include "drmP.h" 29#include "drm.h" 30 31#include "nouveau_drv.h" 32#include "nouveau_vm.h" 33 34#define BAR1_VM_BASE 0x0020000000ULL 35#define BAR1_VM_SIZE pci_resource_len(dev->pdev, 1) 36#define BAR3_VM_BASE 0x0000000000ULL 37#define BAR3_VM_SIZE pci_resource_len(dev->pdev, 3) 38 39struct nv50_instmem_priv { 40 uint32_t save1700[5]; /* 0x1700->0x1710 */ 41 42 struct nouveau_gpuobj *bar1_dmaobj; 43 struct nouveau_gpuobj *bar3_dmaobj; 44}; 45 46static void 47nv50_channel_del(struct nouveau_channel **pchan) 48{ 49 struct nouveau_channel *chan; 50 51 chan = *pchan; 52 *pchan = NULL; 53 if (!chan) 54 return; 55 56 nouveau_gpuobj_ref(NULL, &chan->ramfc); 57 nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd); 58 nouveau_gpuobj_ref(NULL, &chan->vm_pd); 59 if (drm_mm_initialized(&chan->ramin_heap)) 60 drm_mm_takedown(&chan->ramin_heap); 61 nouveau_gpuobj_ref(NULL, &chan->ramin); 62 kfree(chan); 63} 64 65static int 66nv50_channel_new(struct drm_device *dev, u32 size, struct nouveau_vm *vm, 67 struct nouveau_channel **pchan) 68{ 69 struct drm_nouveau_private *dev_priv = dev->dev_private; 70 u32 pgd = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200; 71 u32 fc = (dev_priv->chipset == 0x50) ? 0x0000 : 0x4200; 72 struct nouveau_channel *chan; 73 int ret, i; 74 75 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 76 if (!chan) 77 return -ENOMEM; 78 chan->dev = dev; 79 80 ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin); 81 if (ret) { 82 nv50_channel_del(&chan); 83 return ret; 84 } 85 86 ret = drm_mm_init(&chan->ramin_heap, 0x6000, chan->ramin->size); 87 if (ret) { 88 nv50_channel_del(&chan); 89 return ret; 90 } 91 92 ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 : 93 chan->ramin->pinst + pgd, 94 chan->ramin->vinst + pgd, 95 0x4000, NVOBJ_FLAG_ZERO_ALLOC, 96 &chan->vm_pd); 97 if (ret) { 98 nv50_channel_del(&chan); 99 return ret; 100 } 101 102 for (i = 0; i < 0x4000; i += 8) { 103 nv_wo32(chan->vm_pd, i + 0, 0x00000000); 104 nv_wo32(chan->vm_pd, i + 4, 0xdeadcafe); 105 } 106 107 ret = nouveau_vm_ref(vm, &chan->vm, chan->vm_pd); 108 if (ret) { 109 nv50_channel_del(&chan); 110 return ret; 111 } 112 113 ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 : 114 chan->ramin->pinst + fc, 115 chan->ramin->vinst + fc, 0x100, 116 NVOBJ_FLAG_ZERO_ALLOC, &chan->ramfc); 117 if (ret) { 118 nv50_channel_del(&chan); 119 return ret; 120 } 121 122 *pchan = chan; 123 return 0; 124} 125 126int 127nv50_instmem_init(struct drm_device *dev) 128{ 129 struct drm_nouveau_private *dev_priv = dev->dev_private; 130 struct nv50_instmem_priv *priv; 131 struct nouveau_channel *chan; 132 struct nouveau_vm *vm; 133 int ret, i; 134 u32 tmp; 135 136 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 137 if (!priv) 138 return -ENOMEM; 139 dev_priv->engine.instmem.priv = priv; 140 141 /* Save state, will restore at takedown. */ 142 for (i = 0x1700; i <= 0x1710; i += 4) 143 priv->save1700[(i-0x1700)/4] = nv_rd32(dev, i); 144 145 /* Global PRAMIN heap */ 146 ret = drm_mm_init(&dev_priv->ramin_heap, 0, dev_priv->ramin_size); 147 if (ret) { 148 NV_ERROR(dev, "Failed to init RAMIN heap\n"); 149 goto error; 150 } 151 152 /* BAR3 */ 153 ret = nouveau_vm_new(dev, BAR3_VM_BASE, BAR3_VM_SIZE, BAR3_VM_BASE, 154 &dev_priv->bar3_vm); 155 if (ret) 156 goto error; 157 158 ret = nouveau_gpuobj_new(dev, NULL, (BAR3_VM_SIZE >> 12) * 8, 159 0x1000, NVOBJ_FLAG_DONT_MAP | 160 NVOBJ_FLAG_ZERO_ALLOC, 161 &dev_priv->bar3_vm->pgt[0].obj[0]); 162 if (ret) 163 goto error; 164 dev_priv->bar3_vm->pgt[0].refcount[0] = 1; 165 166 nv50_instmem_map(dev_priv->bar3_vm->pgt[0].obj[0]); 167 168 ret = nv50_channel_new(dev, 128 * 1024, dev_priv->bar3_vm, &chan); 169 if (ret) 170 goto error; 171 dev_priv->channels.ptr[0] = dev_priv->channels.ptr[127] = chan; 172 173 ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR3_VM_BASE, BAR3_VM_SIZE, 174 NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM, 175 NV_MEM_TYPE_VM, NV_MEM_COMP_VM, 176 &priv->bar3_dmaobj); 177 if (ret) 178 goto error; 179 180 nv_wr32(dev, 0x001704, 0x00000000 | (chan->ramin->vinst >> 12)); 181 nv_wr32(dev, 0x001704, 0x40000000 | (chan->ramin->vinst >> 12)); 182 nv_wr32(dev, 0x00170c, 0x80000000 | (priv->bar3_dmaobj->cinst >> 4)); 183 184 dev_priv->engine.instmem.flush(dev); 185 dev_priv->ramin_available = true; 186 187 tmp = nv_ro32(chan->ramin, 0); 188 nv_wo32(chan->ramin, 0, ~tmp); 189 if (nv_ro32(chan->ramin, 0) != ~tmp) { 190 NV_ERROR(dev, "PRAMIN readback failed\n"); 191 ret = -EIO; 192 goto error; 193 } 194 nv_wo32(chan->ramin, 0, tmp); 195 196 /* BAR1 */ 197 ret = nouveau_vm_new(dev, BAR1_VM_BASE, BAR1_VM_SIZE, BAR1_VM_BASE, &vm); 198 if (ret) 199 goto error; 200 201 ret = nouveau_vm_ref(vm, &dev_priv->bar1_vm, chan->vm_pd); 202 if (ret) 203 goto error; 204 nouveau_vm_ref(NULL, &vm, NULL); 205 206 ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR1_VM_BASE, BAR1_VM_SIZE, 207 NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM, 208 NV_MEM_TYPE_VM, NV_MEM_COMP_VM, 209 &priv->bar1_dmaobj); 210 if (ret) 211 goto error; 212 213 nv_wr32(dev, 0x001708, 0x80000000 | (priv->bar1_dmaobj->cinst >> 4)); 214 for (i = 0; i < 8; i++) 215 nv_wr32(dev, 0x1900 + (i*4), 0); 216 217 /* Create shared channel VM, space is reserved at the beginning 218 * to catch "NULL pointer" references 219 */ 220 ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0020000000ULL, 221 &dev_priv->chan_vm); 222 if (ret) 223 return ret; 224 225 return 0; 226 227error: 228 nv50_instmem_takedown(dev); 229 return ret; 230} 231 232void 233nv50_instmem_takedown(struct drm_device *dev) 234{ 235 struct drm_nouveau_private *dev_priv = dev->dev_private; 236 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv; 237 struct nouveau_channel *chan = dev_priv->channels.ptr[0]; 238 int i; 239 240 NV_DEBUG(dev, "\n"); 241 242 if (!priv) 243 return; 244 245 dev_priv->ramin_available = false; 246 247 nouveau_vm_ref(NULL, &dev_priv->chan_vm, NULL); 248 249 for (i = 0x1700; i <= 0x1710; i += 4) 250 nv_wr32(dev, i, priv->save1700[(i - 0x1700) / 4]); 251 252 nouveau_gpuobj_ref(NULL, &priv->bar3_dmaobj); 253 nouveau_gpuobj_ref(NULL, &priv->bar1_dmaobj); 254 255 nouveau_vm_ref(NULL, &dev_priv->bar1_vm, chan->vm_pd); 256 dev_priv->channels.ptr[127] = 0; 257 nv50_channel_del(&dev_priv->channels.ptr[0]); 258 259 nouveau_gpuobj_ref(NULL, &dev_priv->bar3_vm->pgt[0].obj[0]); 260 nouveau_vm_ref(NULL, &dev_priv->bar3_vm, NULL); 261 262 if (drm_mm_initialized(&dev_priv->ramin_heap)) 263 drm_mm_takedown(&dev_priv->ramin_heap); 264 265 dev_priv->engine.instmem.priv = NULL; 266 kfree(priv); 267} 268 269int 270nv50_instmem_suspend(struct drm_device *dev) 271{ 272 struct drm_nouveau_private *dev_priv = dev->dev_private; 273 274 dev_priv->ramin_available = false; 275 return 0; 276} 277 278void 279nv50_instmem_resume(struct drm_device *dev) 280{ 281 struct drm_nouveau_private *dev_priv = dev->dev_private; 282 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv; 283 struct nouveau_channel *chan = dev_priv->channels.ptr[0]; 284 int i; 285 286 /* Poke the relevant regs, and pray it works :) */ 287 nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12)); 288 nv_wr32(dev, NV50_PUNK_UNK1710, 0); 289 nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12) | 290 NV50_PUNK_BAR_CFG_BASE_VALID); 291 nv_wr32(dev, NV50_PUNK_BAR1_CTXDMA, (priv->bar1_dmaobj->cinst >> 4) | 292 NV50_PUNK_BAR1_CTXDMA_VALID); 293 nv_wr32(dev, NV50_PUNK_BAR3_CTXDMA, (priv->bar3_dmaobj->cinst >> 4) | 294 NV50_PUNK_BAR3_CTXDMA_VALID); 295 296 for (i = 0; i < 8; i++) 297 nv_wr32(dev, 0x1900 + (i*4), 0); 298 299 dev_priv->ramin_available = true; 300} 301 302struct nv50_gpuobj_node { 303 struct nouveau_mem *vram; 304 struct nouveau_vma chan_vma; 305 u32 align; 306}; 307 308int 309nv50_instmem_get(struct nouveau_gpuobj *gpuobj, struct nouveau_channel *chan, 310 u32 size, u32 align) 311{ 312 struct drm_device *dev = gpuobj->dev; 313 struct drm_nouveau_private *dev_priv = dev->dev_private; 314 struct nouveau_vram_engine *vram = &dev_priv->engine.vram; 315 struct nv50_gpuobj_node *node = NULL; 316 int ret; 317 318 node = kzalloc(sizeof(*node), GFP_KERNEL); 319 if (!node) 320 return -ENOMEM; 321 node->align = align; 322 323 size = (size + 4095) & ~4095; 324 align = max(align, (u32)4096); 325 326 ret = vram->get(dev, size, align, 0, 0, &node->vram); 327 if (ret) { 328 kfree(node); 329 return ret; 330 } 331 332 gpuobj->vinst = node->vram->offset; 333 334 if (gpuobj->flags & NVOBJ_FLAG_VM) { 335 u32 flags = NV_MEM_ACCESS_RW; 336 if (!(gpuobj->flags & NVOBJ_FLAG_VM_USER)) 337 flags |= NV_MEM_ACCESS_SYS; 338 339 ret = nouveau_vm_get(chan->vm, size, 12, flags, 340 &node->chan_vma); 341 if (ret) { 342 vram->put(dev, &node->vram); 343 kfree(node); 344 return ret; 345 } 346 347 nouveau_vm_map(&node->chan_vma, node->vram); 348 gpuobj->linst = node->chan_vma.offset; 349 } 350 351 gpuobj->size = size; 352 gpuobj->node = node; 353 return 0; 354} 355 356void 357nv50_instmem_put(struct nouveau_gpuobj *gpuobj) 358{ 359 struct drm_device *dev = gpuobj->dev; 360 struct drm_nouveau_private *dev_priv = dev->dev_private; 361 struct nouveau_vram_engine *vram = &dev_priv->engine.vram; 362 struct nv50_gpuobj_node *node; 363 364 node = gpuobj->node; 365 gpuobj->node = NULL; 366 367 if (node->chan_vma.node) { 368 nouveau_vm_unmap(&node->chan_vma); 369 nouveau_vm_put(&node->chan_vma); 370 } 371 vram->put(dev, &node->vram); 372 kfree(node); 373} 374 375int 376nv50_instmem_map(struct nouveau_gpuobj *gpuobj) 377{ 378 struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private; 379 struct nv50_gpuobj_node *node = gpuobj->node; 380 int ret; 381 382 ret = nouveau_vm_get(dev_priv->bar3_vm, gpuobj->size, 12, 383 NV_MEM_ACCESS_RW, &node->vram->bar_vma); 384 if (ret) 385 return ret; 386 387 nouveau_vm_map(&node->vram->bar_vma, node->vram); 388 gpuobj->pinst = node->vram->bar_vma.offset; 389 return 0; 390} 391 392void 393nv50_instmem_unmap(struct nouveau_gpuobj *gpuobj) 394{ 395 struct nv50_gpuobj_node *node = gpuobj->node; 396 397 if (node->vram->bar_vma.node) { 398 nouveau_vm_unmap(&node->vram->bar_vma); 399 nouveau_vm_put(&node->vram->bar_vma); 400 } 401} 402 403void 404nv50_instmem_flush(struct drm_device *dev) 405{ 406 struct drm_nouveau_private *dev_priv = dev->dev_private; 407 unsigned long flags; 408 409 spin_lock_irqsave(&dev_priv->vm_lock, flags); 410 nv_wr32(dev, 0x00330c, 0x00000001); 411 if (!nv_wait(dev, 0x00330c, 0x00000002, 0x00000000)) 412 NV_ERROR(dev, "PRAMIN flush timeout\n"); 413 spin_unlock_irqrestore(&dev_priv->vm_lock, flags); 414} 415 416void 417nv84_instmem_flush(struct drm_device *dev) 418{ 419 struct drm_nouveau_private *dev_priv = dev->dev_private; 420 unsigned long flags; 421 422 spin_lock_irqsave(&dev_priv->vm_lock, flags); 423 nv_wr32(dev, 0x070000, 0x00000001); 424 if (!nv_wait(dev, 0x070000, 0x00000002, 0x00000000)) 425 NV_ERROR(dev, "PRAMIN flush timeout\n"); 426 spin_unlock_irqrestore(&dev_priv->vm_lock, flags); 427} 428 429