/*	$OpenBSD: tlbhandler.S,v 1.53 2021/10/07 13:08:17 visa Exp $ */

/*
 * Copyright (c) 1995-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 */

/*
 *  This code handles TLB exceptions and updates.
 */

#include <machine/param.h>
#include <machine/pte.h>
#include <machine/asm.h>
#include <machine/cpu.h>
#include <mips64/mips_cpu.h>
#include <machine/regnum.h>
#include <machine/cpustate.h>
#ifdef CPU_LOONGSON2
#include <machine/loongson2.h>
#endif

#include "assym.h"

	.set	mips3

	.set	noreorder	/* Default reorder mode */

/*---------------------------------------------------------------- tlb_miss
 *	Low level TLB exception handler. TLB and XTLB code are
 *	identical at the moment.
 *	This code is not copied to the exception area, instead
 *	trampolines branching to the appropriate code are built.
 */
	.set	noat

	.globl	xtlb_miss
	.ent	xtlb_miss, 0
xtlb_miss:
	dmfc0	k0, COP_0_BAD_VADDR
	bltz	k0, _k_miss		# kernel address space
	PTR_SRL	k0, k0, SEGSHIFT
	sltiu	k1, k0, PMAP_SEGTABSIZE
	beqz	k1, _inv_seg		# wrong if outside pm_segtab
	nop
	GET_CPU_INFO(k1, k0)
	PTR_L	k1, CI_CURPROCPADDR(k1)
	dmfc0	k0, COP_0_BAD_VADDR
	PTR_SRL	k0, k0, SEGSHIFT
	PTR_SLL	k0, k0, LOGREGSZ
	PTR_L	k1, PCB_SEGTAB(k1)
	PTR_ADDU k1, k1, k0
	PTR_L	k1, 0(k1)		# get pointer to page directory
	dmfc0	k0, COP_0_BAD_VADDR
	PTR_SRL	k0, k0, (DIRSHIFT - LOGREGSZ)
	beqz	k1, _inv_seg
	andi	k0, k0, (NPDEPG - 1) << LOGREGSZ
	PTR_ADDU k1, k1, k0
	PTR_L	k1, 0(k1)		# get pointer to page table
	dmfc0	k0, COP_0_BAD_VADDR
	PTR_SRL	k0, k0, PGSHIFT - PTE_LOG
	beqz	k1, _inv_seg
	andi	k0, k0, ((NPTEPG >> 1) - 1) << (PTE_LOG + 1)
	PTR_ADDU k1, k1, k0
	PTE_LOAD	k0, 0(k1)
	PTE_LOAD	k1, PTE_OFFS(k1)
	PTE_CLEAR_SWBITS(k0)
	PTE_CLEAR_SWBITS(k1)
	dmtc0	k0, COP_0_TLB_LO0
	dmtc0	k1, COP_0_TLB_LO1
	TLB_HAZARD
	tlbwr				# update TLB
	TLB_HAZARD

#ifdef CPU_LOONGSON2
	li	k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
	dmtc0	k0, COP_0_DIAG
#endif

	ERET

_k_miss:
	dmfc0	k0, COP_0_BAD_VADDR	# must reload.
	b	k_tlb_miss		# kernel tlbmiss.
	nop

_inv_seg:				# No page table for this segment,
					# or processing is too complex to be
					# done here: invoke C code.
	mfc0	k0, COP_0_STATUS_REG
	andi	k0, SR_KSU_USER
	bne	k0, zero, go_u_general
	nop

go_k_general:
	j	k_general
	nop
go_u_general:
#ifdef CPU_LOONGSON2
	/*
	 * To work around a branch prediction issue on earlier LS2F
	 * chips, it is necessary to clear the BTB upon
	 * userland->kernel boundaries.
	 */
	li	k0, COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
	dmtc0	k0, COP_0_DIAG
#endif
	j	u_general
	nop
	.end	xtlb_miss

	.set	at

/*---------------------------------------------------------------- k_tlb_inv
 *	Handle a TLB invalid exception from kernel mode in kernel
 *	space. This happens when we have a TLB match but an invalid
 *	entry. Try to reload.
 */
NLEAF(k_tlb_inv, 0)
	.set	noat
#ifdef CPU_OCTEON
	/* Check if the fault was caused by an instruction fetch. */
	dmfc0	k0, COP_0_CAUSE_REG
	/* XXX bbit0 k0, CR_BR_DELAY_BIT, 1f */
	li	k1, CR_BR_DELAY
	and	k0, k0, k1
	beq	k0, zero, 1f			# fault in a branch delay slot?
	 dmfc0	k1, COP_0_EXC_PC
	daddu	k1, k1, 4			# adjust for the slot
1:	dmfc0	k0, COP_0_BAD_VADDR
	beq	k0, k1, go_k_general
#else
	dmfc0	k0, COP_0_BAD_VADDR		# get the fault address
#endif
	LA	k1, (VM_MIN_KERNEL_ADDRESS)	# compute index
	PTR_SUBU k0, k0, k1
	lw	k1, Sysmapsize			# index within range?
	PTR_SRL	k0, k0, PGSHIFT
	sltu	k1, k0, k1
	beq	k1, zero, sys_stk_chk		# No. check for valid stack
	nop

	PTR_L	k1, Sysmap
	PTR_SLL	k0, k0, PTE_LOG			# compute offset from index
	TLB_HAZARD
	tlbp					# Probe the invalid entry
	TLB_HAZARD				# necessary?
	PTR_ADDU k1, k1, k0
	and	k0, k0, PTE_OFFS		# check even/odd page
	bne	k0, zero, k_tlb_inv_odd
	nop

	mfc0	k0, COP_0_TLB_INDEX
	bltz	k0, sys_stk_chk			# probe fail
	PTE_LOAD k0, 0(k1)			# get PTE entry

	PTE_CLEAR_SWBITS(k0)
	dmtc0	k0, COP_0_TLB_LO0		# load PTE entry
	and	k0, k0, PG_V			# check for valid entry
	beq	k0, zero, go_k_general		# PTE invalid
	PTE_LOAD k0, PTE_OFFS(k1)		# get odd PTE entry
	PTE_CLEAR_SWBITS(k0)
	dmtc0	k0, COP_0_TLB_LO1		# load PTE entry
	TLB_HAZARD
	tlbwi					# write TLB
	TLB_HAZARD

#ifdef CPU_LOONGSON2
	li	k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
	dmtc0	k0, COP_0_DIAG
#endif

	ERET

k_tlb_inv_odd:
	mfc0	k0, COP_0_TLB_INDEX
	bltz	k0, sys_stk_chk			# probe fail
	PTE_LOAD k0, 0(k1)			# get PTE entry

	PTE_CLEAR_SWBITS(k0)
	dmtc0	k0, COP_0_TLB_LO1		# save PTE entry
	and	k0, k0, PG_V			# check for valid entry
	beq	k0, zero, go_k_general		# PTE invalid
	PTE_LOAD k0, -PTE_OFFS(k1)		# get even PTE entry
	PTE_CLEAR_SWBITS(k0)
	dmtc0	k0, COP_0_TLB_LO0		# save PTE entry
	TLB_HAZARD
	tlbwi					# update TLB
	TLB_HAZARD

#ifdef CPU_LOONGSON2
	li	k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
	dmtc0	k0, COP_0_DIAG
#endif

	ERET
END(k_tlb_inv)

/*---------------------------------------------------------------- k_tlb_miss
 *
 *	Handle a TLB miss exception from kernel mode in kernel space.
 *	We must check that this is coming from kernel mode. If not
 *	it's a bad address from user mode so handle properly.
 *	Load up the correct entry contents from the kernel map.
 *	k0 has bad address.
 */
NLEAF(k_tlb_miss, 0)
	.set	noat
	mfc0	k1, COP_0_STATUS_REG
	andi	k1, SR_KSU_USER
	bne	k1, zero, go_u_general
	LA	k1, (VM_MIN_KERNEL_ADDRESS)	# compute index
						# (safe if expands to > 1 insn)
	PTR_SUBU k0, k0, k1
	lw	k1, Sysmapsize			# index within range?
	PTR_SRL	k0, k0, PGSHIFT
	sltu	k1, k0, k1
	beq	k1, zero, sys_stk_chk		# No. check for valid stack
	PTR_SRL	k0, k0, 1
	PTR_L	k1, Sysmap
	PTR_SLL	k0, k0, (PTE_LOG + 1)		# compute offset from index
	PTR_ADDU k1, k1, k0
	PTE_LOAD k0, 0(k1)			# get PTE entry
	PTE_LOAD k1, PTE_OFFS(k1)		# get odd PTE entry
	PTE_CLEAR_SWBITS(k0)
	dmtc0	k0, COP_0_TLB_LO0		# load PTE entry
	PTE_CLEAR_SWBITS(k1)
	dmtc0	k1, COP_0_TLB_LO1		# load PTE entry
	TLB_HAZARD
	tlbwr					# write TLB
	TLB_HAZARD

#ifdef CPU_LOONGSON2
	li	k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
	dmtc0	k0, COP_0_DIAG
#endif

	ERET

sys_stk_chk:
	GET_CPU_INFO(k1, k0)
	PTR_L	k1, CI_CURPROCPADDR(k1)
	PTR_SUBU k0, sp, k1			# check to see if we have a
	sltiu	k0, k0, 2048			#  valid kernel stack
	beqz	k0, go_k_general		# yes, handle.
	nop

	LA	a0, start-FRAMESZ(CF_SZ)-4*REGSZ # set sp to a valid place
#ifdef __mips_n64
	mfc0	a4, COP_0_STATUS_REG
	mfc0	a5, COP_0_CAUSE_REG
	move	a6, sp
#else
	mfc0	a2, COP_0_STATUS_REG
	mfc0	a3, COP_0_CAUSE_REG
	REG_S	a2, CF_ARGSZ+0*REGSZ(sp)
	REG_S	a3, CF_ARGSZ+1*REGSZ(sp)
	PTR_S	sp, CF_ARGSZ+2*REGSZ(a0)
#endif
	move	sp, a0
	dmfc0	a3, COP_0_BAD_VADDR
	move	a2, ra
	dmfc0	a1, COP_0_EXC_PC
	LA	a0, 1f
	jal	printf
	nop

	LA	sp, start-FRAMESZ(CF_SZ)	# set sp to a valid place

#ifdef DDB
	LA	a1, db_printf
	LA	a0, 2f
	jal	trapDump
	nop
#endif

	PANIC("kernel stack overflow")
	/*noreturn*/

	.data
1:
	.asciiz	"\rktlbmiss: PC %p RA %p ADR %p\nSR %p CR %p SP %p\n"
2:
	.asciiz	"stack ovf"
	.text

	.set	at
END(k_tlb_miss)

/*---------------------------------------------------------------- tlb_flush
 *	Flush the "random" entries from the TLB.
 *	Uses "wired" register to determine what register to start with.
 *	Arg "tlbsize" is the number of entries to flush.
 */
LEAF(tlb_flush, 0)
	mfc0	v1, COP_0_STATUS_REG	# Save the status register.
	ori	v0, v1, SR_INT_ENAB
	xori	v0, v0, SR_INT_ENAB
	mtc0	v0, COP_0_STATUS_REG	# Disable interrupts
	MTC0_SR_IE_HAZARD
	mfc0	ta1, COP_0_TLB_WIRED
	LA	v0, CKSEG0_BASE		# invalid address
	dmfc0	ta0, COP_0_TLB_HI	# Save the PID

#ifdef CPU_OCTEON
	PTR_SLL	ta2, ta1, PAGE_SHIFT + 1
	PTR_ADDU v0, v0, ta2
#else
	dmtc0	v0, COP_0_TLB_HI	# Mark entry high as invalid
#endif
	dmtc0	zero, COP_0_TLB_LO0	# Zero out low entry0.
	dmtc0	zero, COP_0_TLB_LO1	# Zero out low entry1.
	mtc0	zero, COP_0_TLB_PG_MASK	# Zero out mask entry.
/*
 * Align the starting value (ta1) and the upper bound (a0).
 */
1:
	mtc0	ta1, COP_0_TLB_INDEX	# Set the index register.
#ifdef CPU_OCTEON
	dmtc0	v0, COP_0_TLB_HI	# Mark entry high as invalid
	li	ta2, 2 * PAGE_SIZE
	PTR_ADDU v0, v0, ta2
#endif
	addu	ta1, ta1, 1		# Increment index.
	TLB_HAZARD
	tlbwi				# Write the TLB entry.
	TLB_HAZARD
	bne	ta1, a0, 1b
	nop

#ifdef CPU_LOONGSON2
	li	v0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
	dmtc0	v0, COP_0_DIAG
#endif

	dmtc0	ta0, COP_0_TLB_HI	# Restore the PID
	li	a0, TLB_PAGE_MASK
	mtc0	a0, COP_0_TLB_PG_MASK	# Restore default mask value.
	mtc0	v1, COP_0_STATUS_REG	# Restore the status register
	MTC0_SR_IE_HAZARD
	j	ra
	nop
END(tlb_flush)

/*--------------------------------------------------------------- tlb_flush_addr
 *	Flush any TLB entries for the given address and TLB PID.
 */
LEAF(tlb_flush_addr, 0)
	mfc0	v1, COP_0_STATUS_REG	# Save the status register.
	ori	v0, v1, SR_INT_ENAB
	xori	v0, v0, SR_INT_ENAB
	mtc0	v0, COP_0_STATUS_REG	# Disable interrupts
	MTC0_SR_IE_HAZARD
	dli	v0, (PG_HVPN | PG_ASID_MASK)
	and	a0, a0, v0		# Make sure valid hi value.
	dmfc0	ta0, COP_0_TLB_HI	# Get current PID
	dmtc0	a0, COP_0_TLB_HI	# look for addr & PID
	TLB_HAZARD
	tlbp				# Probe for the entry.
	TLB_HAZARD			# necessary?
	LA	ta1, CKSEG0_BASE	# Load invalid entry.
	mfc0	v0, COP_0_TLB_INDEX	# See what we got
	bltz	v0, 1f			# index < 0 => !found
	nop
#ifdef CPU_OCTEON
	PTR_SLL	ta2, v0, PAGE_SHIFT + 1
	PTR_ADDU ta1, ta1, ta2
#endif
	dmtc0	ta1, COP_0_TLB_HI	# Mark entry high as invalid

	dmtc0	zero, COP_0_TLB_LO0	# Zero out low entry.
	dmtc0	zero, COP_0_TLB_LO1	# Zero out low entry.
	TLB_HAZARD
	tlbwi
	TLB_HAZARD

#ifdef CPU_LOONGSON2
	li	v0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
	dmtc0	v0, COP_0_DIAG
#endif

1:
	dmtc0	ta0, COP_0_TLB_HI	# restore PID
	mtc0	v1, COP_0_STATUS_REG	# Restore the status register
	MTC0_SR_IE_HAZARD
	j	ra
	nop
END(tlb_flush_addr)

/*---------------------------------------------------------------- tlb_update
 *	Update the TLB if highreg is found; otherwise, enter the data.
 */
LEAF(tlb_update, 0)
	mfc0	v1, COP_0_STATUS_REG	# Save the status register.
	ori	v0, v1, SR_INT_ENAB
	xori	v0, v0, SR_INT_ENAB
	mtc0	v0, COP_0_STATUS_REG	# Disable interrupts
	MTC0_SR_IE_HAZARD
	and	ta1, a0, PG_ODDPG	# ta1 = Even/Odd flag
	dli	v0, (PG_HVPN | PG_ASID_MASK)
	and	a0, a0, v0
	dmfc0	ta0, COP_0_TLB_HI	# Save current PID
	dmtc0	a0, COP_0_TLB_HI	# Init high reg
	and	a2, a1, PG_G		# Copy global bit
	li	a3, TLB_PAGE_MASK
	TLB_HAZARD
	tlbp				# Probe for the entry.
	TLB_HAZARD			# necessary?
	PTE_CLEAR_SWBITS(a1)
	mfc0	v0, COP_0_TLB_INDEX	# See what we got
	bne	ta1, zero, 2f		# Decide even odd
	nop
# EVEN
	bltz	v0, 1f			# index < 0 => !found
	nop

	TLB_HAZARD
	tlbr				# update, read entry first
	TLB_HAZARD			# necessary?
	dmtc0	a1, COP_0_TLB_LO0	# init low reg0.
	TLB_HAZARD
	tlbwi				# update slot found
	TLB_HAZARD
	b	4f
	li	v0, 1
1:
	mtc0	a3, COP_0_TLB_PG_MASK	# init mask.
	dmtc0	a0, COP_0_TLB_HI	# init high reg.
	dmtc0	a1, COP_0_TLB_LO0	# init low reg0.
	dmtc0	a2, COP_0_TLB_LO1	# init low reg1.
	TLB_HAZARD
	tlbwr				# enter into a random slot
	TLB_HAZARD
	b	4f
	li	v0, 0
# ODD
2:
	nop
	bltz	v0, 3f			# index < 0 => !found
	nop

	TLB_HAZARD
	tlbr				# read the entry first
	TLB_HAZARD			# necessary?
	dmtc0	a1, COP_0_TLB_LO1	# init low reg1.
	TLB_HAZARD
	tlbwi				# update slot found
	TLB_HAZARD
	b	4f
	li	v0, 1
3:
	mtc0	a3, COP_0_TLB_PG_MASK	# init mask.
	dmtc0	a0, COP_0_TLB_HI	# init high reg.
	dmtc0	a2, COP_0_TLB_LO0	# init low reg0.
	dmtc0	a1, COP_0_TLB_LO1	# init low reg1.
	TLB_HAZARD
	tlbwr				# enter into a random slot
	TLB_HAZARD
4:
#ifdef CPU_LOONGSON2
	li	v0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
	dmtc0	v0, COP_0_DIAG
#endif

	dmtc0	ta0, COP_0_TLB_HI	# restore PID
	mtc0	v1, COP_0_STATUS_REG	# Restore the status register
	MTC0_SR_IE_HAZARD
	j	ra
	li	v0, 0
END(tlb_update)

/*---------------------------------------------------------------- tlb_read
 *	Read the TLB entry.
 */
LEAF(tlb_read, 0)
	mfc0	v1, COP_0_STATUS_REG	# Save the status register.
	ori	v0, v1, SR_INT_ENAB
	xori	v0, v0, SR_INT_ENAB
	mtc0	v0, COP_0_STATUS_REG	# Disable interrupts
	MTC0_SR_IE_HAZARD
	dmfc0	v0, COP_0_TLB_HI	# Get current PID

	mtc0	a0, COP_0_TLB_INDEX	# Set the index register
	TLB_HAZARD
	tlbr				# Read from the TLB
	TLB_HAZARD			# necessary?
	mfc0	ta0, COP_0_TLB_PG_MASK	# fetch the size
	dmfc0	ta1, COP_0_TLB_HI	# fetch the hi entry
	dmfc0	ta2, COP_0_TLB_LO0	# See what we got
	dmfc0	ta3, COP_0_TLB_LO1	# See what we got
	dmtc0	v0, COP_0_TLB_HI	# restore PID
	nop
	nop
	nop				# wait for PID active
	li	a0, TLB_PAGE_MASK
	mtc0	a0, COP_0_TLB_PG_MASK	# Restore default mask value.
	mtc0	v1, COP_0_STATUS_REG	# Restore the status register
	MTC0_SR_IE_HAZARD
	sd	ta0, 0(a1)
	sd	ta1, 8(a1)
	sd	ta2, 16(a1)
	j	ra
	sd	ta3, 24(a1)
END(tlb_read)

/*---------------------------------------------------------------- tlb_get_pid
 *	Read the tlb pid value.
 */
LEAF(tlb_get_pid, 0)
	dmfc0	v0, COP_0_TLB_HI	# get PID
	li	v1, PG_ASID_MASK	# mask off PID
	j	ra
	and	v0, v0, v1		# mask off PID
END(tlb_get_pid)

/*---------------------------------------------------------------- tlb_set_pid
 *	Write the given pid into the TLB pid reg.
 */
LEAF(tlb_set_pid, 0)
	dmtc0	a0, COP_0_TLB_HI		# Write the hi reg value
	TLB_HAZARD
	j	ra
	nop
END(tlb_set_pid)

/*---------------------------------------------------------------- tlb_set_wired
 *	Write the given value into the TLB wired reg.
 */
LEAF(tlb_set_wired, 0)
	mtc0	a0, COP_0_TLB_WIRED
	j	ra
	nop
END(tlb_set_wired)

/*
 *	Initialize the TLB page mask.
 */
LEAF(tlb_set_page_mask, 0)
	mtc0	a0, COP_0_TLB_PG_MASK
	TLB_HAZARD
	j	ra
	nop
END(tlb_set_page_mask)
