summaryrefslogtreecommitdiffstats
path: root/arch/arm64/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm64/kernel')
-rw-r--r--arch/arm64/kernel/Makefile3
-rw-r--r--arch/arm64/kernel/entry-fpsimd.S43
-rw-r--r--arch/arm64/kernel/entry.S4
-rw-r--r--arch/arm64/kernel/head.S33
-rw-r--r--arch/arm64/kernel/hyp-stub.S109
-rw-r--r--arch/arm64/kernel/signal.c49
-rw-r--r--arch/arm64/kernel/signal32.c20
-rw-r--r--arch/arm64/kernel/vdso.c2
-rw-r--r--arch/arm64/kernel/vdso/gettimeofday.S100
9 files changed, 227 insertions, 136 deletions
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index e2caff1b812..74239c31e25 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -8,7 +8,8 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
# Object file lists.
arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \
entry-fpsimd.o process.o ptrace.o setup.o signal.o \
- sys.o stacktrace.o time.o traps.o io.o vdso.o
+ sys.o stacktrace.o time.o traps.o io.o vdso.o \
+ hyp-stub.o
arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
sys_compat.o
diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S
index 17988a6e7ea..6a27cd6dbfa 100644
--- a/arch/arm64/kernel/entry-fpsimd.S
+++ b/arch/arm64/kernel/entry-fpsimd.S
@@ -20,6 +20,7 @@
#include <linux/linkage.h>
#include <asm/assembler.h>
+#include <asm/fpsimdmacros.h>
/*
* Save the FP registers.
@@ -27,26 +28,7 @@
* x0 - pointer to struct fpsimd_state
*/
ENTRY(fpsimd_save_state)
- stp q0, q1, [x0, #16 * 0]
- stp q2, q3, [x0, #16 * 2]
- stp q4, q5, [x0, #16 * 4]
- stp q6, q7, [x0, #16 * 6]
- stp q8, q9, [x0, #16 * 8]
- stp q10, q11, [x0, #16 * 10]
- stp q12, q13, [x0, #16 * 12]
- stp q14, q15, [x0, #16 * 14]
- stp q16, q17, [x0, #16 * 16]
- stp q18, q19, [x0, #16 * 18]
- stp q20, q21, [x0, #16 * 20]
- stp q22, q23, [x0, #16 * 22]
- stp q24, q25, [x0, #16 * 24]
- stp q26, q27, [x0, #16 * 26]
- stp q28, q29, [x0, #16 * 28]
- stp q30, q31, [x0, #16 * 30]!
- mrs x8, fpsr
- str w8, [x0, #16 * 2]
- mrs x8, fpcr
- str w8, [x0, #16 * 2 + 4]
+ fpsimd_save x0, 8
ret
ENDPROC(fpsimd_save_state)
@@ -56,25 +38,6 @@ ENDPROC(fpsimd_save_state)
* x0 - pointer to struct fpsimd_state
*/
ENTRY(fpsimd_load_state)
- ldp q0, q1, [x0, #16 * 0]
- ldp q2, q3, [x0, #16 * 2]
- ldp q4, q5, [x0, #16 * 4]
- ldp q6, q7, [x0, #16 * 6]
- ldp q8, q9, [x0, #16 * 8]
- ldp q10, q11, [x0, #16 * 10]
- ldp q12, q13, [x0, #16 * 12]
- ldp q14, q15, [x0, #16 * 14]
- ldp q16, q17, [x0, #16 * 16]
- ldp q18, q19, [x0, #16 * 18]
- ldp q20, q21, [x0, #16 * 20]
- ldp q22, q23, [x0, #16 * 22]
- ldp q24, q25, [x0, #16 * 24]
- ldp q26, q27, [x0, #16 * 26]
- ldp q28, q29, [x0, #16 * 28]
- ldp q30, q31, [x0, #16 * 30]!
- ldr w8, [x0, #16 * 2]
- ldr w9, [x0, #16 * 2 + 4]
- msr fpsr, x8
- msr fpcr, x9
+ fpsimd_restore x0, 8
ret
ENDPROC(fpsimd_load_state)
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index cbfa4d28100..9c94f404ded 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -148,10 +148,6 @@ tsk .req x28 // current thread_info
/*
* Exception vectors.
*/
- .macro ventry label
- .align 7
- b \label
- .endm
.align 11
ENTRY(vectors)
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index a2f02b63eae..368ad1f7c36 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -31,6 +31,7 @@
#include <asm/pgtable-hwdef.h>
#include <asm/pgtable.h>
#include <asm/page.h>
+#include <asm/virt.h>
/*
* swapper_pg_dir is the virtual address of the initial page table. We place
@@ -115,13 +116,13 @@
ENTRY(stext)
mov x21, x0 // x21=FDT
+ bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
bl el2_setup // Drop to EL1
mrs x22, midr_el1 // x22=cpuid
mov x0, x22
bl lookup_processor_type
mov x23, x0 // x23=current cpu_table
cbz x23, __error_p // invalid processor (x23=0)?
- bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
bl __vet_fdt
bl __create_page_tables // x25=TTBR0, x26=TTBR1
/*
@@ -147,17 +148,23 @@ ENTRY(el2_setup)
mrs x0, CurrentEL
cmp x0, #PSR_MODE_EL2t
ccmp x0, #PSR_MODE_EL2h, #0x4, ne
+ ldr x0, =__boot_cpu_mode // Compute __boot_cpu_mode
+ add x0, x0, x28
b.eq 1f
+ str wzr, [x0] // Remember we don't have EL2...
ret
/* Hyp configuration. */
-1: mov x0, #(1 << 31) // 64-bit EL1
+1: ldr w1, =BOOT_CPU_MODE_EL2
+ str w1, [x0, #4] // This CPU has EL2
+ mov x0, #(1 << 31) // 64-bit EL1
msr hcr_el2, x0
/* Generic timers. */
mrs x0, cnthctl_el2
orr x0, x0, #3 // Enable EL1 physical timers
msr cnthctl_el2, x0
+ msr cntvoff_el2, xzr // Clear virtual offset
/* Populate ID registers. */
mrs x0, midr_el1
@@ -178,6 +185,13 @@ ENTRY(el2_setup)
msr hstr_el2, xzr // Disable CP15 traps to EL2
#endif
+ /* Stage-2 translation */
+ msr vttbr_el2, xzr
+
+ /* Hypervisor stub */
+ adr x0, __hyp_stub_vectors
+ msr vbar_el2, x0
+
/* spsr */
mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
PSR_MODE_EL1h)
@@ -186,6 +200,19 @@ ENTRY(el2_setup)
eret
ENDPROC(el2_setup)
+/*
+ * We need to find out the CPU boot mode long after boot, so we need to
+ * store it in a writable variable.
+ *
+ * This is not in .bss, because we set it sufficiently early that the boot-time
+ * zeroing of .bss would clobber it.
+ */
+ .pushsection .data
+ENTRY(__boot_cpu_mode)
+ .long BOOT_CPU_MODE_EL2
+ .long 0
+ .popsection
+
.align 3
2: .quad .
.quad PAGE_OFFSET
@@ -201,6 +228,7 @@ ENDPROC(el2_setup)
* cores are held until we're ready for them to initialise.
*/
ENTRY(secondary_holding_pen)
+ bl __calc_phys_offset // x24=phys offset
bl el2_setup // Drop to EL1
mrs x0, mpidr_el1
and x0, x0, #15 // CPU number
@@ -226,7 +254,6 @@ ENTRY(secondary_startup)
mov x23, x0 // x23=current cpu_table
cbz x23, __error_p // invalid processor (x23=0)?
- bl __calc_phys_offset // x24=phys offset
pgtbl x25, x26, x24 // x25=TTBR0, x26=TTBR1
ldr x12, [x23, #CPU_INFO_SETUP]
add x12, x12, x28 // __virt_to_phys
diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S
new file mode 100644
index 00000000000..0959611d9ff
--- /dev/null
+++ b/arch/arm64/kernel/hyp-stub.S
@@ -0,0 +1,109 @@
+/*
+ * Hypervisor stub
+ *
+ * Copyright (C) 2012 ARM Ltd.
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/init.h>
+#include <linux/linkage.h>
+
+#include <asm/assembler.h>
+#include <asm/ptrace.h>
+#include <asm/virt.h>
+
+ .text
+ .align 11
+
+ENTRY(__hyp_stub_vectors)
+ ventry el2_sync_invalid // Synchronous EL2t
+ ventry el2_irq_invalid // IRQ EL2t
+ ventry el2_fiq_invalid // FIQ EL2t
+ ventry el2_error_invalid // Error EL2t
+
+ ventry el2_sync_invalid // Synchronous EL2h
+ ventry el2_irq_invalid // IRQ EL2h
+ ventry el2_fiq_invalid // FIQ EL2h
+ ventry el2_error_invalid // Error EL2h
+
+ ventry el1_sync // Synchronous 64-bit EL1
+ ventry el1_irq_invalid // IRQ 64-bit EL1
+ ventry el1_fiq_invalid // FIQ 64-bit EL1
+ ventry el1_error_invalid // Error 64-bit EL1
+
+ ventry el1_sync_invalid // Synchronous 32-bit EL1
+ ventry el1_irq_invalid // IRQ 32-bit EL1
+ ventry el1_fiq_invalid // FIQ 32-bit EL1
+ ventry el1_error_invalid // Error 32-bit EL1
+ENDPROC(__hyp_stub_vectors)
+
+ .align 11
+
+el1_sync:
+ mrs x1, esr_el2
+ lsr x1, x1, #26
+ cmp x1, #0x16
+ b.ne 2f // Not an HVC trap
+ cbz x0, 1f
+ msr vbar_el2, x0 // Set vbar_el2
+ b 2f
+1: mrs x0, vbar_el2 // Return vbar_el2
+2: eret
+ENDPROC(el1_sync)
+
+.macro invalid_vector label
+\label:
+ b \label
+ENDPROC(\label)
+.endm
+
+ invalid_vector el2_sync_invalid
+ invalid_vector el2_irq_invalid
+ invalid_vector el2_fiq_invalid
+ invalid_vector el2_error_invalid
+ invalid_vector el1_sync_invalid
+ invalid_vector el1_irq_invalid
+ invalid_vector el1_fiq_invalid
+ invalid_vector el1_error_invalid
+
+/*
+ * __hyp_set_vectors: Call this after boot to set the initial hypervisor
+ * vectors as part of hypervisor installation. On an SMP system, this should
+ * be called on each CPU.
+ *
+ * x0 must be the physical address of the new vector table, and must be
+ * 2KB aligned.
+ *
+ * Before calling this, you must check that the stub hypervisor is installed
+ * everywhere, by waiting for any secondary CPUs to be brought up and then
+ * checking that is_hyp_mode_available() is true.
+ *
+ * If not, there is a pre-existing hypervisor, some CPUs failed to boot, or
+ * something else went wrong... in such cases, trying to install a new
+ * hypervisor is unlikely to work as desired.
+ *
+ * When you call into your shiny new hypervisor, sp_el2 will contain junk,
+ * so you will need to set that to something sensible at the new hypervisor's
+ * initialisation entry point.
+ */
+
+ENTRY(__hyp_get_vectors)
+ mov x0, xzr
+ // fall through
+ENTRY(__hyp_set_vectors)
+ hvc #0
+ ret
+ENDPROC(__hyp_get_vectors)
+ENDPROC(__hyp_set_vectors)
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 8807ba2cf26..abd756315cb 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -41,6 +41,8 @@
struct rt_sigframe {
struct siginfo info;
struct ucontext uc;
+ u64 fp;
+ u64 lr;
};
static int preserve_fpsimd_context(struct fpsimd_context __user *ctx)
@@ -175,6 +177,10 @@ static int setup_sigframe(struct rt_sigframe __user *sf,
struct aux_context __user *aux =
(struct aux_context __user *)sf->uc.uc_mcontext.__reserved;
+ /* set up the stack frame for unwinding */
+ __put_user_error(regs->regs[29], &sf->fp, err);
+ __put_user_error(regs->regs[30], &sf->lr, err);
+
for (i = 0; i < 31; i++)
__put_user_error(regs->regs[i], &sf->uc.uc_mcontext.regs[i],
err);
@@ -196,11 +202,11 @@ static int setup_sigframe(struct rt_sigframe __user *sf,
return err;
}
-static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
- int framesize)
+static struct rt_sigframe __user *get_sigframe(struct k_sigaction *ka,
+ struct pt_regs *regs)
{
unsigned long sp, sp_top;
- void __user *frame;
+ struct rt_sigframe __user *frame;
sp = sp_top = regs->sp;
@@ -210,11 +216,8 @@ static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
if ((ka->sa.sa_flags & SA_ONSTACK) && !sas_ss_flags(sp))
sp = sp_top = current->sas_ss_sp + current->sas_ss_size;
- /* room for stack frame (FP, LR) */
- sp -= 16;
-
- sp = (sp - framesize) & ~15;
- frame = (void __user *)sp;
+ sp = (sp - sizeof(struct rt_sigframe)) & ~15;
+ frame = (struct rt_sigframe __user *)sp;
/*
* Check that we can actually write to the signal frame.
@@ -225,20 +228,14 @@ static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
return frame;
}
-static int setup_return(struct pt_regs *regs, struct k_sigaction *ka,
- void __user *frame, int usig)
+static void setup_return(struct pt_regs *regs, struct k_sigaction *ka,
+ void __user *frame, int usig)
{
- int err = 0;
__sigrestore_t sigtramp;
- unsigned long __user *sp = (unsigned long __user *)regs->sp;
-
- /* set up the stack frame */
- __put_user_error(regs->regs[29], sp - 2, err);
- __put_user_error(regs->regs[30], sp - 1, err);
regs->regs[0] = usig;
- regs->regs[29] = regs->sp - 16;
regs->sp = (unsigned long)frame;
+ regs->regs[29] = regs->sp + offsetof(struct rt_sigframe, fp);
regs->pc = (unsigned long)ka->sa.sa_handler;
if (ka->sa.sa_flags & SA_RESTORER)
@@ -247,8 +244,6 @@ static int setup_return(struct pt_regs *regs, struct k_sigaction *ka,
sigtramp = VDSO_SYMBOL(current->mm->context.vdso, sigtramp);
regs->regs[30] = (unsigned long)sigtramp;
-
- return err;
}
static int setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
@@ -258,7 +253,7 @@ static int setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
stack_t stack;
int err = 0;
- frame = get_sigframe(ka, regs, sizeof(*frame));
+ frame = get_sigframe(ka, regs);
if (!frame)
return 1;
@@ -272,13 +267,13 @@ static int setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
err |= __copy_to_user(&frame->uc.uc_stack, &stack, sizeof(stack));
err |= setup_sigframe(frame, regs, set);
- if (err == 0)
- err = setup_return(regs, ka, frame, usig);
-
- if (err == 0 && ka->sa.sa_flags & SA_SIGINFO) {
- err |= copy_siginfo_to_user(&frame->info, info);
- regs->regs[1] = (unsigned long)&frame->info;
- regs->regs[2] = (unsigned long)&frame->uc;
+ if (err == 0) {
+ setup_return(regs, ka, frame, usig);
+ if (ka->sa.sa_flags & SA_SIGINFO) {
+ err |= copy_siginfo_to_user(&frame->info, info);
+ regs->regs[1] = (unsigned long)&frame->info;
+ regs->regs[2] = (unsigned long)&frame->uc;
+ }
}
return err;
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index 4654824747a..a4db3d22aac 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -578,9 +578,9 @@ badframe:
return 0;
}
-static inline void __user *compat_get_sigframe(struct k_sigaction *ka,
- struct pt_regs *regs,
- int framesize)
+static void __user *compat_get_sigframe(struct k_sigaction *ka,
+ struct pt_regs *regs,
+ int framesize)
{
compat_ulong_t sp = regs->compat_sp;
void __user *frame;
@@ -605,9 +605,9 @@ static inline void __user *compat_get_sigframe(struct k_sigaction *ka,
return frame;
}
-static int compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
- compat_ulong_t __user *rc, void __user *frame,
- int usig)
+static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
+ compat_ulong_t __user *rc, void __user *frame,
+ int usig)
{
compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler);
compat_ulong_t retcode;
@@ -643,8 +643,6 @@ static int compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
regs->compat_lr = retcode;
regs->pc = handler;
regs->pstate = spsr;
-
- return 0;
}
static int compat_setup_sigframe(struct compat_sigframe __user *sf,
@@ -714,11 +712,9 @@ int compat_setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
err |= __copy_to_user(&frame->sig.uc.uc_stack, &stack, sizeof(stack));
err |= compat_setup_sigframe(&frame->sig, regs, set);
- if (err == 0)
- err = compat_setup_return(regs, ka, frame->sig.retcode, frame,
- usig);
if (err == 0) {
+ compat_setup_return(regs, ka, frame->sig.retcode, frame, usig);
regs->regs[1] = (compat_ulong_t)(unsigned long)&frame->info;
regs->regs[2] = (compat_ulong_t)(unsigned long)&frame->sig.uc;
}
@@ -741,7 +737,7 @@ int compat_setup_frame(int usig, struct k_sigaction *ka, sigset_t *set,
err |= compat_setup_sigframe(frame, regs, set);
if (err == 0)
- err = compat_setup_return(regs, ka, frame->retcode, frame, usig);
+ compat_setup_return(regs, ka, frame->retcode, frame, usig);
return err;
}
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index ba457943a16..c958cb84d75 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -239,7 +239,7 @@ void update_vsyscall(struct timekeeper *tk)
if (!use_syscall) {
vdso_data->cs_cycle_last = tk->clock->cycle_last;
vdso_data->xtime_clock_sec = tk->xtime_sec;
- vdso_data->xtime_clock_nsec = tk->xtime_nsec >> tk->shift;
+ vdso_data->xtime_clock_nsec = tk->xtime_nsec;
vdso_data->cs_mult = tk->mult;
vdso_data->cs_shift = tk->shift;
vdso_data->wtm_clock_sec = tk->wall_to_monotonic.tv_sec;
diff --git a/arch/arm64/kernel/vdso/gettimeofday.S b/arch/arm64/kernel/vdso/gettimeofday.S
index dcb8c203a3b..8bf658d974f 100644
--- a/arch/arm64/kernel/vdso/gettimeofday.S
+++ b/arch/arm64/kernel/vdso/gettimeofday.S
@@ -62,18 +62,19 @@ ENTRY(__kernel_gettimeofday)
/* If tv is NULL, skip to the timezone code. */
cbz x0, 2f
bl __do_get_tspec
- seqcnt_check w13, 1b
+ seqcnt_check w9, 1b
/* Convert ns to us. */
- mov x11, #1000
- udiv x10, x10, x11
- stp x9, x10, [x0, #TVAL_TV_SEC]
+ mov x13, #1000
+ lsl x13, x13, x12
+ udiv x11, x11, x13
+ stp x10, x11, [x0, #TVAL_TV_SEC]
2:
/* If tz is NULL, return 0. */
cbz x1, 3f
ldp w4, w5, [vdso_data, #VDSO_TZ_MINWEST]
- seqcnt_read w13
- seqcnt_check w13, 1b
+ seqcnt_read w9
+ seqcnt_check w9, 1b
stp w4, w5, [x1, #TZ_MINWEST]
3:
mov x0, xzr
@@ -102,17 +103,17 @@ ENTRY(__kernel_clock_gettime)
cbnz use_syscall, 7f
bl __do_get_tspec
- seqcnt_check w13, 1b
+ seqcnt_check w9, 1b
cmp w0, #CLOCK_MONOTONIC
b.ne 6f
/* Get wtm timespec. */
- ldp x14, x15, [vdso_data, #VDSO_WTM_CLK_SEC]
+ ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC]
/* Check the sequence counter. */
- seqcnt_read w13
- seqcnt_check w13, 1b
+ seqcnt_read w9
+ seqcnt_check w9, 1b
b 4f
2:
cmp w0, #CLOCK_REALTIME_COARSE
@@ -122,37 +123,40 @@ ENTRY(__kernel_clock_gettime)
/* Get coarse timespec. */
adr vdso_data, _vdso_data
3: seqcnt_acquire
- ldp x9, x10, [vdso_data, #VDSO_XTIME_CRS_SEC]
-
- cmp w0, #CLOCK_MONOTONIC_COARSE
- b.ne 6f
+ ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC]
/* Get wtm timespec. */
- ldp x14, x15, [vdso_data, #VDSO_WTM_CLK_SEC]
+ ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC]
/* Check the sequence counter. */
- seqcnt_read w13
- seqcnt_check w13, 3b
+ seqcnt_read w9
+ seqcnt_check w9, 3b
+
+ cmp w0, #CLOCK_MONOTONIC_COARSE
+ b.ne 6f
4:
/* Add on wtm timespec. */
- add x9, x9, x14
- add x10, x10, x15
+ add x10, x10, x13
+ lsl x14, x14, x12
+ add x11, x11, x14
/* Normalise the new timespec. */
- mov x14, #NSEC_PER_SEC_LO16
- movk x14, #NSEC_PER_SEC_HI16, lsl #16
- cmp x10, x14
+ mov x15, #NSEC_PER_SEC_LO16
+ movk x15, #NSEC_PER_SEC_HI16, lsl #16
+ lsl x15, x15, x12
+ cmp x11, x15
b.lt 5f
- sub x10, x10, x14
- add x9, x9, #1
+ sub x11, x11, x15
+ add x10, x10, #1
5:
- cmp x10, #0
+ cmp x11, #0
b.ge 6f
- add x10, x10, x14
- sub x9, x9, #1
+ add x11, x11, x15
+ sub x10, x10, #1
6: /* Store to the user timespec. */
- stp x9, x10, [x1, #TSPEC_TV_SEC]
+ lsr x11, x11, x12
+ stp x10, x11, [x1, #TSPEC_TV_SEC]
mov x0, xzr
ret x2
7:
@@ -203,39 +207,39 @@ ENDPROC(__kernel_clock_getres)
* Expects vdso_data to be initialised.
* Clobbers the temporary registers (x9 - x15).
* Returns:
- * - (x9, x10) = (ts->tv_sec, ts->tv_nsec)
- * - (x11, x12) = (xtime->tv_sec, xtime->tv_nsec)
- * - w13 = vDSO sequence counter
+ * - w9 = vDSO sequence counter
+ * - (x10, x11) = (ts->tv_sec, shifted ts->tv_nsec)
+ * - w12 = cs_shift
*/
ENTRY(__do_get_tspec)
.cfi_startproc
/* Read from the vDSO data page. */
ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST]
- ldp x11, x12, [vdso_data, #VDSO_XTIME_CLK_SEC]
- ldp w14, w15, [vdso_data, #VDSO_CS_MULT]
- seqcnt_read w13
+ ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC]
+ ldp w11, w12, [vdso_data, #VDSO_CS_MULT]
+ seqcnt_read w9
- /* Read the physical counter. */
+ /* Read the virtual counter. */
isb
- mrs x9, cntpct_el0
+ mrs x15, cntvct_el0
/* Calculate cycle delta and convert to ns. */
- sub x10, x9, x10
+ sub x10, x15, x10
/* We can only guarantee 56 bits of precision. */
- movn x9, #0xff0, lsl #48
- and x10, x9, x10
- mul x10, x10, x14
- lsr x10, x10, x15
+ movn x15, #0xff00, lsl #48
+ and x10, x15, x10
+ mul x10, x10, x11
/* Use the kernel time to calculate the new timespec. */
- add x10, x12, x10
- mov x14, #NSEC_PER_SEC_LO16
- movk x14, #NSEC_PER_SEC_HI16, lsl #16
- udiv x15, x10, x14
- add x9, x15, x11
- mul x14, x14, x15
- sub x10, x10, x14
+ mov x11, #NSEC_PER_SEC_LO16
+ movk x11, #NSEC_PER_SEC_HI16, lsl #16
+ lsl x11, x11, x12
+ add x15, x10, x14
+ udiv x14, x15, x11
+ add x10, x13, x14
+ mul x13, x14, x11
+ sub x11, x15, x13
ret
.cfi_endproc
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-03-01 07:58:37 +0000