1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180

// Copyright (c) 2017 Colin Finck, RWTH Aachen University
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

use crate::arch::x86_64::kernel::irq::IrqStatistics;
use crate::arch::x86_64::kernel::BOOT_INFO;
use crate::scheduler::{CoreId, PerCoreScheduler};
use crate::x86::bits64::task::TaskStateSegment;
use crate::x86::msr::*;
use core::ptr;
use crossbeam_utils::CachePadded;

pub static mut PERCORE: PerCoreVariables = CachePadded::new(PerCoreInnerVariables::new(0));

pub type PerCoreVariables = CachePadded<PerCoreInnerVariables>;

pub struct PerCoreInnerVariables {
	/// Sequential ID of this CPU Core.
	core_id: PerCoreVariable<CoreId>,
	/// Scheduler for this CPU Core.
	scheduler: PerCoreVariable<*mut PerCoreScheduler>,
	/// Task State Segment (TSS) allocated for this CPU Core.
	pub tss: PerCoreVariable<*mut TaskStateSegment>,
	/// start address of the kernel stack
	pub kernel_stack: PerCoreVariable<u64>,
	/// Interface to the interrupt counters
	pub irq_statistics: PerCoreVariable<*mut IrqStatistics>,
}

impl PerCoreInnerVariables {
	pub const fn new(core_id: CoreId) -> Self {
		Self {
			core_id: PerCoreVariable::new(core_id),
			scheduler: PerCoreVariable::new(ptr::null_mut() as *mut PerCoreScheduler),
			tss: PerCoreVariable::new(ptr::null_mut() as *mut TaskStateSegment),
			kernel_stack: PerCoreVariable::new(0),
			irq_statistics: PerCoreVariable::new(ptr::null_mut() as *mut IrqStatistics),
		}
	}
}

#[repr(C)]
pub struct PerCoreVariable<T> {
	data: T,
}

pub trait PerCoreVariableMethods<T> {
	unsafe fn get(&self) -> T
	where
		T: Copy;
	unsafe fn set(&self, value: T);
}

impl<T> PerCoreVariable<T> {
	pub const fn new(value: T) -> Self {
		Self { data: value }
	}

	#[inline]
	unsafe fn offset(&self) -> usize {
		let base = &PERCORE as *const _ as usize;
		let field = self as *const _ as usize;
		field - base
	}
}

// Treat all per-core variables as 64-bit variables by default. This is true for u64, usize, pointers.
// Implement the PerCoreVariableMethods trait functions using 64-bit memory moves.
// The functions are implemented as default functions, which can be overridden in specialized implementations of the trait.
impl<T> PerCoreVariableMethods<T> for PerCoreVariable<T> {
	#[inline]
	#[cfg(feature = "smp")]
	default unsafe fn get(&self) -> T
	where
		T: Copy,
	{
		let value: T;
		llvm_asm!("movq %gs:($1), $0" : "=r"(value) : "r"(self.offset()) :: "volatile");
		value
	}

	#[inline]
	#[cfg(not(feature = "smp"))]
	default unsafe fn get(&self) -> T
	where
		T: Copy,
	{
		let value: *const T = core::mem::transmute(&PERCORE as *const _ as usize + self.offset());
		*value
	}

	#[inline]
	#[cfg(feature = "smp")]
	default unsafe fn set(&self, value: T) {
		llvm_asm!("movq $0, %gs:($1)" :: "r"(value), "r"(self.offset()) :: "volatile");
	}

	#[inline]
	#[cfg(not(feature = "smp"))]
	default unsafe fn set(&self, new_value: T) {
		let value: *mut T = core::mem::transmute(&PERCORE as *const _ as usize + self.offset());
		*value = new_value;
	}
}

// Define and implement a trait to mark all 32-bit variables used inside PerCoreVariables.
pub trait Is32BitVariable {}
impl Is32BitVariable for u32 {}

// For all types implementing the Is32BitVariable trait above, implement the PerCoreVariableMethods
// trait functions using 32-bit memory moves.
impl<T: Is32BitVariable> PerCoreVariableMethods<T> for PerCoreVariable<T> {
	#[inline]
	unsafe fn get(&self) -> T {
		let value: T;
		llvm_asm!("movl %gs:($1), $0" : "=r"(value) : "r"(self.offset()) :: "volatile");
		value
	}

	#[inline]
	unsafe fn set(&self, value: T) {
		llvm_asm!("movl $0, %gs:($1)" :: "r"(value), "r"(self.offset()) :: "volatile");
	}
}

#[cfg(target_os = "hermit")]
#[inline]
pub fn core_id() -> CoreId {
	unsafe { PERCORE.core_id.get() }
}

#[cfg(not(target_os = "hermit"))]
pub fn core_id() -> CoreId {
	0
}

#[inline(always)]
pub fn get_kernel_stack() -> u64 {
	unsafe { PERCORE.kernel_stack.get() }
}

#[inline]
pub fn set_kernel_stack(addr: u64) {
	unsafe { PERCORE.kernel_stack.set(addr) }
}

#[inline]
pub fn core_scheduler() -> &'static mut PerCoreScheduler {
	unsafe { &mut *PERCORE.scheduler.get() }
}

#[inline]
pub fn set_core_scheduler(scheduler: *mut PerCoreScheduler) {
	unsafe {
		PERCORE.scheduler.set(scheduler);
	}
}

#[inline]
pub fn increment_irq_counter(irq_no: usize) {
	unsafe {
		let irq = &mut *PERCORE.irq_statistics.get();
		irq.inc(irq_no);
	}
}

pub fn init() {
	unsafe {
		// Store the address to the PerCoreVariables structure allocated for this core in GS.
		let address = core::ptr::read_volatile(&(*BOOT_INFO).current_percore_address);
		if address == 0 {
			wrmsr(IA32_GS_BASE, &PERCORE as *const _ as u64);
		} else {
			wrmsr(IA32_GS_BASE, address as u64);
		}
	}
}