Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PAL/Linux-SGX] AEX-Notify 3/5: Add AEX-Notify enabling code #2034

Open
wants to merge 1 commit into
base: dimakuv/aex-notify-part2
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions Documentation/manifest-syntax.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1392,6 +1392,30 @@ In addition, the application manifest must also contain ``sgx.debug = true``.

See :ref:`vtune-sgx-profiling` for more information.

Enabling AEX-Notify
^^^^^^^^^^^^^^^^^^^

::

sgx.experimental_enable_aex_notify = [true|false]
(Default: false)

When enabled, this option instructs Gramine to use the AEX-Notify hardware feature.
AEX-Notify is a flexible hardware extension that makes SGX enclaves interrupt
aware: enclaves can register a trusted handler to be run after an interrupt or
exception. AEX-Notify can be used as a building block for implementing
countermeasures against different types of interrupt-based attacks in software.

For more information on AEX-Notify, refer to the `academic paper
<https://www.usenix.org/conference/usenixsecurity23/presentation/constable>`__
and to the `official whitepaper
<https://cdrdv2-public.intel.com/736463/aex-notify-white-paper-public.pdf>`__.

.. warning::
Support for AEX-Notify in Gramine is not yet thoroughly tested and may
contain security vulnerabilities. This is temporary; the prefix
``experimental_`` will be removed in the future after thorough validation.

Deprecated options
------------------

Expand Down
4 changes: 4 additions & 0 deletions pal/src/host/linux-sgx/generated_offsets.c
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ const struct generated_offset generated_offsets[] = {
/* defines from sgx_arch.h */
DEFINE(SGX_FLAGS_DEBUG, SGX_FLAGS_DEBUG),
DEFINE(SGX_FLAGS_MODE64BIT, SGX_FLAGS_MODE64BIT),
DEFINE(SGX_FLAGS_AEXNOTIFY, SGX_FLAGS_AEXNOTIFY),
DEFINE(SGX_XFRM_LEGACY, SGX_XFRM_LEGACY),
DEFINE(SGX_XFRM_AVX, SGX_XFRM_AVX),
DEFINE(SGX_XFRM_MPX, SGX_XFRM_MPX),
Expand Down Expand Up @@ -54,6 +55,7 @@ const struct generated_offset generated_offsets[] = {
OFFSET_T(SGX_GPR_RFLAGS, sgx_pal_gpr_t, rflags),
OFFSET_T(SGX_GPR_RIP, sgx_pal_gpr_t, rip),
OFFSET_T(SGX_GPR_EXITINFO, sgx_pal_gpr_t, exitinfo),
OFFSET_T(SGX_GPR_AEXNOTIFY, sgx_pal_gpr_t, aexnotify),
DEFINE(SGX_GPR_SIZE, sizeof(sgx_pal_gpr_t)),

/* sgx_cpu_context_t */
Expand Down Expand Up @@ -103,6 +105,7 @@ const struct generated_offset generated_offsets[] = {
OFFSET(SGX_HEAP_MIN, pal_enclave_tcb, heap_min),
OFFSET(SGX_HEAP_MAX, pal_enclave_tcb, heap_max),
OFFSET(SGX_CLEAR_CHILD_TID, pal_enclave_tcb, clear_child_tid),
OFFSET(SGX_READY_FOR_AEX_NOTIFY, pal_enclave_tcb, ready_for_aex_notify),

/* struct pal_host_tcb aka PAL_HOST_TCB */
OFFSET(PAL_HOST_TCB_TCS, pal_host_tcb, tcs),
Expand All @@ -123,6 +126,7 @@ const struct generated_offset generated_offsets[] = {
OFFSET_T(TCS_OFS_LIMIT, sgx_arch_tcs_t, ofs_limit),
OFFSET_T(TCS_OGS_LIMIT, sgx_arch_tcs_t, ogs_limit),
DEFINE(TCS_SIZE, sizeof(sgx_arch_tcs_t)),
DEFINE(TCS_FLAGS_AEXNOTIFY, TCS_FLAGS_AEXNOTIFY),

/* sgx_attributes_t */
OFFSET_T(SGX_ATTRIBUTES_XFRM, sgx_attributes_t, xfrm),
Expand Down
18 changes: 18 additions & 0 deletions pal/src/host/linux-sgx/host_framework.c
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,24 @@ bool is_wrfsbase_supported(void) {
return true;
}

bool is_aexnotify_supported(void) {
uint32_t cpuinfo[4];

cpuid(INTEL_SGX_LEAF, 1, cpuinfo);
if (!((cpuinfo[CPUID_WORD_EAX] >> 10) & 0x1)) {
log_error("AEX-Notify hardware feature is not supported.");
return false;
}

cpuid(INTEL_SGX_LEAF, 0, cpuinfo);
if (!((cpuinfo[CPUID_WORD_EAX] >> 11) & 0x1)) {
log_error("ENCLU[EDECCSSA] leaf instruction is not supported.");
return false;
}

return true;
}

int create_enclave(sgx_arch_secs_t* secs, sgx_arch_token_t* token) {
assert(secs->size && IS_POWER_OF_2(secs->size));
assert(IS_ALIGNED(secs->base, secs->size));
Expand Down
1 change: 1 addition & 0 deletions pal/src/host/linux-sgx/host_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ void* realloc(void* ptr, size_t new_size);

int open_sgx_driver(void);
bool is_wrfsbase_supported(void);
bool is_aexnotify_supported(void);

int read_enclave_token(int token_file, sgx_arch_token_t* out_token);
int create_dummy_enclave_token(sgx_sigstruct_t* sig, sgx_arch_token_t* out_token);
Expand Down
18 changes: 18 additions & 0 deletions pal/src/host/linux-sgx/host_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,10 @@ static int initialize_enclave(struct pal_enclave* enclave, const char* manifest_
tcs->ogs_base = tls_area->addr - enclave->baseaddr + t * g_page_size;
tcs->ofs_limit = 0xfff;
tcs->ogs_limit = 0xfff;
tcs->flags |= (enclave_token.body.attributes.flags & SGX_FLAGS_AEXNOTIFY)
? TCS_FLAGS_AEXNOTIFY : 0;
tcs_addrs[t] = (void*)tcs_area->addr + g_page_size * t;

}
} else if (areas[i].data_src == BUF) {
memcpy(data, areas[i].buf, areas[i].buf_size);
Expand Down Expand Up @@ -690,6 +693,21 @@ static int parse_loader_config(char* manifest, struct pal_enclave* enclave_info,
goto out;
}

bool aex_notify_enabled;
ret = toml_bool_in(manifest_root, "sgx.experimental_enable_aex_notify",
/*defaultval=*/false, &aex_notify_enabled);
if (ret < 0) {
log_error("Cannot parse 'sgx.experimental_enable_aex_notify'");
ret = -EINVAL;
goto out;
}

if (aex_notify_enabled && !is_aexnotify_supported()) {
log_error("Cannot enable AEX-Notify on this platform (hardware doesn't support it)");
ret = -EPERM;
goto out;
}

int64_t thread_num_int64;
ret = toml_int_in(manifest_root, "sgx.max_threads", /*defaultval=*/-1, &thread_num_int64);
if (ret < 0) {
Expand Down
35 changes: 35 additions & 0 deletions pal/src/host/linux-sgx/pal_exception.c
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,41 @@

#define ADDR_IN_PAL(addr) ((void*)(addr) > TEXT_START && (void*)(addr) < TEXT_END)

bool g_aex_notify_enabled = false;

void init_aex_notify_for_thread(void) {
if (!g_aex_notify_enabled)
return;

SET_ENCLAVE_TCB(ready_for_aex_notify, 1UL);
MB();
#if 0
/*
* FIXME: Re-enable in the following commit, when all AEX-Notify flows are added.
* Currently this would fail, as the untrusted runtime expects AEX-Notify flows but
* in-enclave runtime doesn't yet implement AEX-Notify flows.
*/
/*
* Note that AEX-Notify is enabled only for SSA[0] (regular context), and is always disabled
* for SSA[1] (stage-1 signal handling context). The disablement of AEX-Notify for SSA[1] is
* implicit (the AEX-Notify bit in the SSA[1]'s GPR region is by default zero). This disablement
* is important for stage-1 signal handling flows, see enclave_entry.S.
*/
GET_ENCLAVE_TCB(gpr)->aexnotify = 1U;
MB();
#endif
}

void fini_aex_notify_for_thread(void) {
if (!g_aex_notify_enabled)
return;

SET_ENCLAVE_TCB(ready_for_aex_notify, 0UL);
MB();
GET_ENCLAVE_TCB(gpr)->aexnotify = 0U;
MB();
}

/* Restore an sgx_cpu_context_t as generated by .Lhandle_exception. Execution will
* continue as specified by the rip in the context. */
__attribute_no_sanitize_address
Expand Down
4 changes: 4 additions & 0 deletions pal/src/host/linux-sgx/pal_linux.h
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ void save_xregs(PAL_XREGS_STATE* xsave_area);
void restore_xregs(const PAL_XREGS_STATE* xsave_area);
noreturn void _restore_sgx_context(sgx_cpu_context_t* uc, PAL_XREGS_STATE* xsave_area);

extern bool g_aex_notify_enabled;
void init_aex_notify_for_thread(void);
void fini_aex_notify_for_thread(void);

void _PalExceptionHandler(uint32_t trusted_exit_info_,
uint32_t untrusted_external_event, sgx_cpu_context_t* uc,
PAL_XREGS_STATE* xregs_state, sgx_arch_exinfo_t* exinfo);
Expand Down
9 changes: 9 additions & 0 deletions pal/src/host/linux-sgx/pal_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -722,6 +722,13 @@ noreturn void pal_linux_main(void* uptr_libpal_uri, size_t libpal_uri_len, void*
ocall_exit(1, /*is_exitgroup=*/true);
}

ret = toml_bool_in(g_pal_public_state.manifest_root, "sgx.experimental_enable_aex_notify",
/*defaultval=*/false, &g_aex_notify_enabled);
if (ret < 0) {
log_error("Cannot parse 'sgx.experimental_enable_aex_notify'");
ocall_exit(1, /*is_exitgroup=*/true);
}

/* Get host information about domain name configuration only for the first process.
* This information will be checkpointed and restored during forking of the child
* process(es). */
Expand Down Expand Up @@ -764,6 +771,8 @@ noreturn void pal_linux_main(void* uptr_libpal_uri, size_t libpal_uri_len, void*
assert(!g_pal_linuxsgx_state.enclave_initialized);
g_pal_linuxsgx_state.enclave_initialized = true;

init_aex_notify_for_thread();

/* call main function */
pal_main(instance_id, parent, first_thread, arguments, environments, post_callback);
}
1 change: 1 addition & 0 deletions pal/src/host/linux-sgx/pal_tcb.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ struct pal_enclave_tcb {
uint64_t ocall_exit_called;
uint64_t thread_started;
uint64_t ready_for_exceptions;
uint64_t ready_for_aex_notify;
uint64_t manifest_size;
void* heap_min;
void* heap_max;
Expand Down
5 changes: 5 additions & 0 deletions pal/src/host/linux-sgx/pal_threading.c
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ static void init_dynamic_thread(void* addr) {
tcs->ogs_base = (uint64_t)tcb - g_enclave_base;
tcs->ofs_limit = 0xfff;
tcs->ogs_limit = 0xfff;
tcs->flags |= g_aex_notify_enabled ? TCS_FLAGS_AEXNOTIFY : 0;
}

static int create_dynamic_tcs_if_none_available(void** out_tcs) {
Expand Down Expand Up @@ -160,6 +161,8 @@ void pal_start_thread(void) {
pal_set_tcb_stack_canary(stack_protector_canary);
PAL_TCB* pal_tcb = pal_get_tcb();
memset(&pal_tcb->libos_tcb, 0, sizeof(pal_tcb->libos_tcb));
init_aex_notify_for_thread();

callback((void*)param);
_PalThreadExit(/*clear_child_tid=*/NULL);
/* UNREACHABLE */
Expand Down Expand Up @@ -227,6 +230,8 @@ void _PalThreadYieldExecution(void) {
noreturn void _PalThreadExit(int* clear_child_tid) {
struct pal_handle_thread* exiting_thread = GET_ENCLAVE_TCB(thread);

fini_aex_notify_for_thread();

/* thread is ready to exit, must inform LibOS by erasing clear_child_tid;
* note that we don't do it now (because this thread still occupies SGX
* TCS slot) but during handle_thread_reset in assembly code */
Expand Down
9 changes: 7 additions & 2 deletions pal/src/host/linux-sgx/sgx_arch.h
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ typedef uint8_t sgx_isvfamily_id_t[SGX_ISV_FAMILY_ID_SIZE];
#define SGX_FLAGS_MODE64BIT 0x04ULL
#define SGX_FLAGS_PROVISION_KEY 0x10ULL
#define SGX_FLAGS_LICENSE_KEY 0x20ULL
#define SGX_FLAGS_AEXNOTIFY 0x400ULL

/* EINIT must verify *all* SECS.ATTRIBUTES[63..0] bits (FLAGS bits) against
* SIGSTRUCT.ATTRIBUTES[63..0].
Expand Down Expand Up @@ -168,7 +169,8 @@ typedef struct {
} sgx_arch_tcs_t;
static_assert(sizeof(sgx_arch_tcs_t) == 4096, "incorrect struct size");

#define TCS_FLAGS_DBGOPTIN (01ULL)
#define TCS_FLAGS_DBGOPTIN (01ULL)
#define TCS_FLAGS_AEXNOTIFY (02ULL)

typedef struct {
uint64_t rax;
Expand All @@ -192,10 +194,12 @@ typedef struct {
uint64_t ursp;
uint64_t urbp;
uint32_t exitinfo;
uint32_t reserved;
uint8_t reserved[3];
uint8_t aexnotify;
uint64_t fsbase;
uint64_t gsbase;
} sgx_pal_gpr_t;
static_assert(offsetof(sgx_pal_gpr_t, aexnotify) == 167, "Wrong offset of AEX-Notify bit in SSA");

typedef struct {
uint64_t rax;
Expand Down Expand Up @@ -443,6 +447,7 @@ static inline int enclu(uint32_t eax, uint64_t rbx, uint64_t rcx, uint64_t rdx)
#define EACCEPT 5
#define EMODPE 6
#define EACCEPTCOPY 7
#define EDECCSSA 9

#define SGX_LAUNCH_KEY 0
#define SGX_PROVISION_KEY 1
Expand Down
1 change: 1 addition & 0 deletions python/graminelibos/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,7 @@ def __init__(self, manifest_str):
sgx.setdefault('debug', False)
sgx.setdefault('enable_stats', False)
sgx.setdefault('edmm_enable', False)
sgx.setdefault('experimental_enable_aex_notify', False)

if sgx['edmm_enable']:
sgx.setdefault('enclave_size', DEFAULT_ENCLAVE_SIZE_WITH_EDMM)
Expand Down
1 change: 1 addition & 0 deletions python/graminelibos/manifest_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@
},
'debug': bool,
'edmm_enable': bool,
'experimental_enable_aex_notify': bool,
'enable_stats': bool,
'enclave_size': _size,
'file_check_policy': Any('strict', 'allow_all_but_log'),
Expand Down
6 changes: 6 additions & 0 deletions python/graminelibos/sgx_sign.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ def collect_cpu_feature_bits(manifest_cpu_features, options_dict, val, mask, sec
def get_enclave_attributes(manifest_sgx):
flags_dict = {
'debug': offs.SGX_FLAGS_DEBUG,
'experimental_enable_aex_notify': offs.SGX_FLAGS_AEXNOTIFY,
}
flags = collect_bits(manifest_sgx, flags_dict)
if ARCHITECTURE == 'amd64':
Expand Down Expand Up @@ -271,6 +272,9 @@ def set_tls_field(t, offset, value):
set_tcs_field(t, offs.TCS_OFS_LIMIT, '<L', 0xfff)
set_tcs_field(t, offs.TCS_OGS_LIMIT, '<L', 0xfff)

if attr['enable_aex_notify']:
set_tcs_field(t, offs.TCS_FLAGS, '<Q', offs.TCS_FLAGS_AEXNOTIFY)

set_tls_field(t, offs.SGX_COMMON_SELF, tls_area.addr + offs.PAGESIZE * t)
set_tls_field(t, offs.SGX_COMMON_STACK_PROTECTOR_CANARY,
offs.STACK_PROTECTOR_CANARY_DEFAULT)
Expand Down Expand Up @@ -473,13 +477,15 @@ def get_mrenclave_and_manifest(manifest_path, libpal, verbose=False):
attr = {
'enclave_size': parse_size(manifest_sgx['enclave_size']),
'edmm_enable': manifest_sgx.get('edmm_enable', False),
'enable_aex_notify': manifest_sgx.get('experimental_enable_aex_notify', False),
'max_threads': manifest_sgx['max_threads'],
}

if verbose:
print('Attributes (required for enclave measurement):')
print(f' size: {attr["enclave_size"]:#x}')
print(f' edmm: {attr["edmm_enable"]}')
print(f' aex-notify: {attr["enable_aex_notify"]}')
print(f' max_threads: {attr["max_threads"]}')

print('SGX remote attestation:')
Expand Down