From f204e0b8cedd7da1dfcfd05ed6b7692737e24029 Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Wed, 8 Oct 2014 19:55:02 +1100 Subject: cxl: Driver code for powernv PCIe based cards for userspace access This is the core of the cxl driver. It adds support for using cxl cards in the powernv environment only (ie POWER8 bare metal). It allows access to cxl accelerators by userspace using the /dev/cxl/afuM.N char devices. The kernel driver has no knowledge of the function implemented by the accelerator. It provides services to userspace via the /dev/cxl/afuM.N devices. When a program opens this device and runs the start work IOCTL, the accelerator will have coherent access to that processes memory using the same virtual addresses. That process may mmap the device to access any MMIO space the accelerator provides. Also, reads on the device will allow interrupts to be received. These services are further documented in a later patch in Documentation/powerpc/cxl.txt. Documentation of the cxl hardware architecture and userspace API is provided in subsequent patches. Signed-off-by: Ian Munsie Signed-off-by: Michael Neuling Signed-off-by: Michael Ellerman --- drivers/misc/cxl/fault.c | 291 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 drivers/misc/cxl/fault.c (limited to 'drivers/misc/cxl/fault.c') diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c new file mode 100644 index 00000000000..69506ebd4d0 --- /dev/null +++ b/drivers/misc/cxl/fault.c @@ -0,0 +1,291 @@ +/* + * Copyright 2014 IBM Corp. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#include +#include +#include +#include +#include + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "cxl" "." +#include +#include +#include + +#include "cxl.h" + +static struct cxl_sste* find_free_sste(struct cxl_sste *primary_group, + bool sec_hash, + struct cxl_sste *secondary_group, + unsigned int *lru) +{ + unsigned int i, entry; + struct cxl_sste *sste, *group = primary_group; + + for (i = 0; i < 2; i++) { + for (entry = 0; entry < 8; entry++) { + sste = group + entry; + if (!(be64_to_cpu(sste->esid_data) & SLB_ESID_V)) + return sste; + } + if (!sec_hash) + break; + group = secondary_group; + } + /* Nothing free, select an entry to cast out */ + if (sec_hash && (*lru & 0x8)) + sste = secondary_group + (*lru & 0x7); + else + sste = primary_group + (*lru & 0x7); + *lru = (*lru + 1) & 0xf; + + return sste; +} + +static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb) +{ + /* mask is the group index, we search primary and secondary here. */ + unsigned int mask = (ctx->sst_size >> 7)-1; /* SSTP0[SegTableSize] */ + bool sec_hash = 1; + struct cxl_sste *sste; + unsigned int hash; + unsigned long flags; + + + sec_hash = !!(cxl_p1n_read(ctx->afu, CXL_PSL_SR_An) & CXL_PSL_SR_An_SC); + + if (slb->vsid & SLB_VSID_B_1T) + hash = (slb->esid >> SID_SHIFT_1T) & mask; + else /* 256M */ + hash = (slb->esid >> SID_SHIFT) & mask; + + spin_lock_irqsave(&ctx->sste_lock, flags); + sste = find_free_sste(ctx->sstp + (hash << 3), sec_hash, + ctx->sstp + ((~hash & mask) << 3), &ctx->sst_lru); + + pr_devel("CXL Populating SST[%li]: %#llx %#llx\n", + sste - ctx->sstp, slb->vsid, slb->esid); + + sste->vsid_data = cpu_to_be64(slb->vsid); + sste->esid_data = cpu_to_be64(slb->esid); + spin_unlock_irqrestore(&ctx->sste_lock, flags); +} + +static int cxl_fault_segment(struct cxl_context *ctx, struct mm_struct *mm, + u64 ea) +{ + struct copro_slb slb = {0,0}; + int rc; + + if (!(rc = copro_calculate_slb(mm, ea, &slb))) { + cxl_load_segment(ctx, &slb); + } + + return rc; +} + +static void cxl_ack_ae(struct cxl_context *ctx) +{ + unsigned long flags; + + cxl_ack_irq(ctx, CXL_PSL_TFC_An_AE, 0); + + spin_lock_irqsave(&ctx->lock, flags); + ctx->pending_fault = true; + ctx->fault_addr = ctx->dar; + ctx->fault_dsisr = ctx->dsisr; + spin_unlock_irqrestore(&ctx->lock, flags); + + wake_up_all(&ctx->wq); +} + +static int cxl_handle_segment_miss(struct cxl_context *ctx, + struct mm_struct *mm, u64 ea) +{ + int rc; + + pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx->pe, ea); + + if ((rc = cxl_fault_segment(ctx, mm, ea))) + cxl_ack_ae(ctx); + else { + + mb(); /* Order seg table write to TFC MMIO write */ + cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0); + } + + return IRQ_HANDLED; +} + +static void cxl_handle_page_fault(struct cxl_context *ctx, + struct mm_struct *mm, u64 dsisr, u64 dar) +{ + unsigned flt = 0; + int result; + unsigned long access, flags; + + if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) { + pr_devel("copro_handle_mm_fault failed: %#x\n", result); + return cxl_ack_ae(ctx); + } + + /* + * update_mmu_cache() will not have loaded the hash since current->trap + * is not a 0x400 or 0x300, so just call hash_page_mm() here. + */ + access = _PAGE_PRESENT; + if (dsisr & CXL_PSL_DSISR_An_S) + access |= _PAGE_RW; + if ((!ctx->kernel) || ~(dar & (1ULL << 63))) + access |= _PAGE_USER; + local_irq_save(flags); + hash_page_mm(mm, dar, access, 0x300); + local_irq_restore(flags); + + pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe); + cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0); +} + +void cxl_handle_fault(struct work_struct *fault_work) +{ + struct cxl_context *ctx = + container_of(fault_work, struct cxl_context, fault_work); + u64 dsisr = ctx->dsisr; + u64 dar = ctx->dar; + struct task_struct *task; + struct mm_struct *mm; + + if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr || + cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An) != dar || + cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) != ctx->pe) { + /* Most likely explanation is harmless - a dedicated process + * has detached and these were cleared by the PSL purge, but + * warn about it just in case */ + dev_notice(&ctx->afu->dev, "cxl_handle_fault: Translation fault regs changed\n"); + return; + } + + pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. " + "DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar); + + if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { + pr_devel("cxl_handle_fault unable to get task %i\n", + pid_nr(ctx->pid)); + cxl_ack_ae(ctx); + return; + } + if (!(mm = get_task_mm(task))) { + pr_devel("cxl_handle_fault unable to get mm %i\n", + pid_nr(ctx->pid)); + cxl_ack_ae(ctx); + goto out; + } + + if (dsisr & CXL_PSL_DSISR_An_DS) + cxl_handle_segment_miss(ctx, mm, dar); + else if (dsisr & CXL_PSL_DSISR_An_DM) + cxl_handle_page_fault(ctx, mm, dsisr, dar); + else + WARN(1, "cxl_handle_fault has nothing to handle\n"); + + mmput(mm); +out: + put_task_struct(task); +} + +static void cxl_prefault_one(struct cxl_context *ctx, u64 ea) +{ + int rc; + struct task_struct *task; + struct mm_struct *mm; + + if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { + pr_devel("cxl_prefault_one unable to get task %i\n", + pid_nr(ctx->pid)); + return; + } + if (!(mm = get_task_mm(task))) { + pr_devel("cxl_prefault_one unable to get mm %i\n", + pid_nr(ctx->pid)); + put_task_struct(task); + return; + } + + rc = cxl_fault_segment(ctx, mm, ea); + + mmput(mm); + put_task_struct(task); +} + +static u64 next_segment(u64 ea, u64 vsid) +{ + if (vsid & SLB_VSID_B_1T) + ea |= (1ULL << 40) - 1; + else + ea |= (1ULL << 28) - 1; + + return ea + 1; +} + +static void cxl_prefault_vma(struct cxl_context *ctx) +{ + u64 ea, last_esid = 0; + struct copro_slb slb; + struct vm_area_struct *vma; + int rc; + struct task_struct *task; + struct mm_struct *mm; + + if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { + pr_devel("cxl_prefault_vma unable to get task %i\n", + pid_nr(ctx->pid)); + return; + } + if (!(mm = get_task_mm(task))) { + pr_devel("cxl_prefault_vm unable to get mm %i\n", + pid_nr(ctx->pid)); + goto out1; + } + + down_read(&mm->mmap_sem); + for (vma = mm->mmap; vma; vma = vma->vm_next) { + for (ea = vma->vm_start; ea < vma->vm_end; + ea = next_segment(ea, slb.vsid)) { + rc = copro_calculate_slb(mm, ea, &slb); + if (rc) + continue; + + if (last_esid == slb.esid) + continue; + + cxl_load_segment(ctx, &slb); + last_esid = slb.esid; + } + } + up_read(&mm->mmap_sem); + + mmput(mm); +out1: + put_task_struct(task); +} + +void cxl_prefault(struct cxl_context *ctx, u64 wed) +{ + switch (ctx->afu->prefault_mode) { + case CXL_PREFAULT_WED: + cxl_prefault_one(ctx, wed); + break; + case CXL_PREFAULT_ALL: + cxl_prefault_vma(ctx); + break; + default: + break; + } +} -- cgit v1.2.3-70-g09d2 From 5100a9d6444bf205de49190431b0d08de43b86e9 Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Tue, 28 Oct 2014 14:25:27 +1100 Subject: cxl: Disable secondary hash in segment table This patch simplifies the process of finding a free segment table entry by disabling the secondary hash. This reduces the number of possible entries in the segment table for a given address from 16 to 8. Due to the large segment sizes we use it is extremely unlikely that the secondary hash would ever have been used in practice, so this should not have any negative impacts and may even improve performance due to the reduced number of comparisons that software & hardware need to perform. This patch clears the SC bit in the hardware's state register (CXL_PSL_SR_An) to disable the secondary hash in the hardware since we can no longer fill out entries using it. Signed-off-by: Ian Munsie Reviewed-by: Aneesh Kumar K.V Signed-off-by: Michael Ellerman --- drivers/misc/cxl/fault.c | 30 ++++++++---------------------- drivers/misc/cxl/native.c | 4 ++-- 2 files changed, 10 insertions(+), 24 deletions(-) (limited to 'drivers/misc/cxl/fault.c') diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c index 69506ebd4d0..d0e97fd3d4c 100644 --- a/drivers/misc/cxl/fault.c +++ b/drivers/misc/cxl/fault.c @@ -22,29 +22,19 @@ #include "cxl.h" static struct cxl_sste* find_free_sste(struct cxl_sste *primary_group, - bool sec_hash, - struct cxl_sste *secondary_group, unsigned int *lru) { - unsigned int i, entry; + unsigned int entry; struct cxl_sste *sste, *group = primary_group; - for (i = 0; i < 2; i++) { - for (entry = 0; entry < 8; entry++) { - sste = group + entry; - if (!(be64_to_cpu(sste->esid_data) & SLB_ESID_V)) - return sste; - } - if (!sec_hash) - break; - group = secondary_group; + for (entry = 0; entry < 8; entry++) { + sste = group + entry; + if (!(be64_to_cpu(sste->esid_data) & SLB_ESID_V)) + return sste; } /* Nothing free, select an entry to cast out */ - if (sec_hash && (*lru & 0x8)) - sste = secondary_group + (*lru & 0x7); - else - sste = primary_group + (*lru & 0x7); - *lru = (*lru + 1) & 0xf; + sste = primary_group + *lru; + *lru = (*lru + 1) & 0x7; return sste; } @@ -53,22 +43,18 @@ static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb) { /* mask is the group index, we search primary and secondary here. */ unsigned int mask = (ctx->sst_size >> 7)-1; /* SSTP0[SegTableSize] */ - bool sec_hash = 1; struct cxl_sste *sste; unsigned int hash; unsigned long flags; - sec_hash = !!(cxl_p1n_read(ctx->afu, CXL_PSL_SR_An) & CXL_PSL_SR_An_SC); - if (slb->vsid & SLB_VSID_B_1T) hash = (slb->esid >> SID_SHIFT_1T) & mask; else /* 256M */ hash = (slb->esid >> SID_SHIFT) & mask; spin_lock_irqsave(&ctx->sste_lock, flags); - sste = find_free_sste(ctx->sstp + (hash << 3), sec_hash, - ctx->sstp + ((~hash & mask) << 3), &ctx->sst_lru); + sste = find_free_sste(ctx->sstp + (hash << 3), &ctx->sst_lru); pr_devel("CXL Populating SST[%li]: %#llx %#llx\n", sste - ctx->sstp, slb->vsid, slb->esid); diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c index 623286a7711..d47532e8f4f 100644 --- a/drivers/misc/cxl/native.c +++ b/drivers/misc/cxl/native.c @@ -417,7 +417,7 @@ static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr) ctx->elem->haurp = 0; /* disable */ ctx->elem->sdr = cpu_to_be64(mfspr(SPRN_SDR1)); - sr = CXL_PSL_SR_An_SC; + sr = 0; if (ctx->master) sr |= CXL_PSL_SR_An_MP; if (mfspr(SPRN_LPCR) & LPCR_TC) @@ -508,7 +508,7 @@ static int attach_dedicated(struct cxl_context *ctx, u64 wed, u64 amr) u64 sr; int rc; - sr = CXL_PSL_SR_An_SC; + sr = 0; set_endian(sr); if (ctx->master) sr |= CXL_PSL_SR_An_MP; -- cgit v1.2.3-70-g09d2 From b03a7f578ac831276fe3870ebda01609d18167de Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Tue, 28 Oct 2014 14:25:28 +1100 Subject: cxl: Refactor cxl_load_segment() and find_free_sste() This moves the segment table hash calculation from cxl_load_segment() into find_free_sste() since that is the only place it is actually used. Signed-off-by: Ian Munsie Reviewed-by: Aneesh Kumar K.V Signed-off-by: Michael Ellerman --- drivers/misc/cxl/fault.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'drivers/misc/cxl/fault.c') diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c index d0e97fd3d4c..cb4f3231b45 100644 --- a/drivers/misc/cxl/fault.c +++ b/drivers/misc/cxl/fault.c @@ -21,20 +21,30 @@ #include "cxl.h" -static struct cxl_sste* find_free_sste(struct cxl_sste *primary_group, - unsigned int *lru) +/* This finds a free SSTE for the given SLB */ +static struct cxl_sste* find_free_sste(struct cxl_context *ctx, + struct copro_slb *slb) { + struct cxl_sste *primary, *sste; + unsigned int mask = (ctx->sst_size >> 7) - 1; /* SSTP0[SegTableSize] */ unsigned int entry; - struct cxl_sste *sste, *group = primary_group; + unsigned int hash; + + if (slb->vsid & SLB_VSID_B_1T) + hash = (slb->esid >> SID_SHIFT_1T) & mask; + else /* 256M */ + hash = (slb->esid >> SID_SHIFT) & mask; - for (entry = 0; entry < 8; entry++) { - sste = group + entry; + primary = ctx->sstp + (hash << 3); + + for (entry = 0, sste = primary; entry < 8; entry++, sste++) { if (!(be64_to_cpu(sste->esid_data) & SLB_ESID_V)) return sste; } + /* Nothing free, select an entry to cast out */ - sste = primary_group + *lru; - *lru = (*lru + 1) & 0x7; + sste = primary + ctx->sst_lru; + ctx->sst_lru = (ctx->sst_lru + 1) & 0x7; return sste; } @@ -42,19 +52,11 @@ static struct cxl_sste* find_free_sste(struct cxl_sste *primary_group, static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb) { /* mask is the group index, we search primary and secondary here. */ - unsigned int mask = (ctx->sst_size >> 7)-1; /* SSTP0[SegTableSize] */ struct cxl_sste *sste; - unsigned int hash; unsigned long flags; - - if (slb->vsid & SLB_VSID_B_1T) - hash = (slb->esid >> SID_SHIFT_1T) & mask; - else /* 256M */ - hash = (slb->esid >> SID_SHIFT) & mask; - spin_lock_irqsave(&ctx->sste_lock, flags); - sste = find_free_sste(ctx->sstp + (hash << 3), &ctx->sst_lru); + sste = find_free_sste(ctx, slb); pr_devel("CXL Populating SST[%li]: %#llx %#llx\n", sste - ctx->sstp, slb->vsid, slb->esid); -- cgit v1.2.3-70-g09d2 From eb01d4c2388ce3b5bcc120d0f72912117ed7599d Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Tue, 28 Oct 2014 14:25:30 +1100 Subject: cxl: Fix PSL error due to duplicate segment table entries In certain circumstances the PSL (Power Service Layer, which provides translation services for CXL hardware) can send an interrupt for a segment miss that the kernel has already handled. This can happen if multiple translations for the same segment are queued in the PSL before the kernel has restarted the first translation. The CXL driver does not expect this situation and does not check if a segment had already been handled. This could cause a duplicate segment table entry which in turn caused a PSL error taking down the card. This patch fixes the issue by checking for existing entries in the segment table that match the segment we are trying to insert, so as to avoid inserting duplicate entries. Signed-off-by: Ian Munsie Reviewed-by: Aneesh Kumar K.V Signed-off-by: Michael Ellerman --- drivers/misc/cxl/fault.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) (limited to 'drivers/misc/cxl/fault.c') diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c index cb4f3231b45..c99e896604e 100644 --- a/drivers/misc/cxl/fault.c +++ b/drivers/misc/cxl/fault.c @@ -21,11 +21,20 @@ #include "cxl.h" -/* This finds a free SSTE for the given SLB */ +static bool sste_matches(struct cxl_sste *sste, struct copro_slb *slb) +{ + return ((sste->vsid_data == cpu_to_be64(slb->vsid)) && + (sste->esid_data == cpu_to_be64(slb->esid))); +} + +/* + * This finds a free SSTE for the given SLB, or returns NULL if it's already in + * the segment table. + */ static struct cxl_sste* find_free_sste(struct cxl_context *ctx, struct copro_slb *slb) { - struct cxl_sste *primary, *sste; + struct cxl_sste *primary, *sste, *ret = NULL; unsigned int mask = (ctx->sst_size >> 7) - 1; /* SSTP0[SegTableSize] */ unsigned int entry; unsigned int hash; @@ -38,15 +47,19 @@ static struct cxl_sste* find_free_sste(struct cxl_context *ctx, primary = ctx->sstp + (hash << 3); for (entry = 0, sste = primary; entry < 8; entry++, sste++) { - if (!(be64_to_cpu(sste->esid_data) & SLB_ESID_V)) - return sste; + if (!ret && !(be64_to_cpu(sste->esid_data) & SLB_ESID_V)) + ret = sste; + if (sste_matches(sste, slb)) + return NULL; } + if (ret) + return ret; /* Nothing free, select an entry to cast out */ - sste = primary + ctx->sst_lru; + ret = primary + ctx->sst_lru; ctx->sst_lru = (ctx->sst_lru + 1) & 0x7; - return sste; + return ret; } static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb) @@ -57,12 +70,15 @@ static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb) spin_lock_irqsave(&ctx->sste_lock, flags); sste = find_free_sste(ctx, slb); + if (!sste) + goto out_unlock; pr_devel("CXL Populating SST[%li]: %#llx %#llx\n", sste - ctx->sstp, slb->vsid, slb->esid); sste->vsid_data = cpu_to_be64(slb->vsid); sste->esid_data = cpu_to_be64(slb->esid); +out_unlock: spin_unlock_irqrestore(&ctx->sste_lock, flags); } -- cgit v1.2.3-70-g09d2