Skip to content

Commit

Permalink
cxl/pci: Add RCH downstream port AER register discovery
Browse files Browse the repository at this point in the history
Restricted CXL host (RCH) downstream port AER information is not currently
logged while in the error state. One problem preventing the error logging
is the AER and RAS registers are not accessible. The CXL driver requires
changes to find RCH downstream port AER and RAS registers for purpose of
error logging.

RCH downstream ports are not enumerated during a PCI bus scan and are
instead discovered using system firmware, ACPI in this case.[1] The
downstream port is implemented as a Root Complex Register Block (RCRB).
The RCRB is a 4k memory block containing PCIe registers based on the PCIe
root port.[2] The RCRB includes AER extended capability registers used for
reporting errors. Note, the RCH's AER Capability is located in the RCRB
memory space instead of PCI configuration space, thus its register access
is different. Existing kernel PCIe AER functions can not be used to manage
the downstream port AER capabilities and RAS registers because the port was
not enumerated during PCI scan and the registers are not PCI config
accessible.

Discover RCH downstream port AER extended capability registers. Use MMIO
accesses to search for extended AER capability in RCRB register space.

[1] CXL 3.0 Spec, 9.11.2 - System Firmware View of CXL 1.1 Hierarchy
[2] CXL 3.0 Spec, 8.2.1.1 - RCH Downstream Port RCRB

Co-developed-by: Robert Richter <[email protected]>
Signed-off-by: Terry Bowman <[email protected]>
Signed-off-by: Robert Richter <[email protected]>
Reviewed-by: Jonathan Cameron <[email protected]>
Reviewed-by: Dave Jiang <[email protected]>
Link: https://lore.kernel.org/r/[email protected]
Signed-off-by: Dan Williams <[email protected]>
  • Loading branch information
Robert Richter authored and djbw committed Oct 28, 2023
1 parent a2fcb84 commit f05fd10
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 0 deletions.
1 change: 1 addition & 0 deletions drivers/cxl/core/core.h
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ struct cxl_rcrb_info;
resource_size_t __rcrb_to_component(struct device *dev,
struct cxl_rcrb_info *ri,
enum cxl_rcrb which);
u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb);

extern struct rw_semaphore cxl_dpa_rwsem;

Expand Down
15 changes: 15 additions & 0 deletions drivers/cxl/core/pci.c
Original file line number Diff line number Diff line change
Expand Up @@ -718,6 +718,21 @@ static bool cxl_report_and_clear(struct cxl_dev_state *cxlds)
return true;
}

#ifdef CONFIG_PCIEAER_CXL

void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport)
{
struct device *dport_dev = dport->dport_dev;
struct pci_host_bridge *host_bridge;

host_bridge = to_pci_host_bridge(dport_dev);
if (host_bridge->native_cxl_error)
dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base);
}
EXPORT_SYMBOL_NS_GPL(cxl_setup_parent_dport, CXL);

#endif

pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
pci_channel_state_t state)
{
Expand Down
36 changes: 36 additions & 0 deletions drivers/cxl/core/regs.c
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,42 @@ int cxl_setup_regs(struct cxl_register_map *map)
}
EXPORT_SYMBOL_NS_GPL(cxl_setup_regs, CXL);

u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb)
{
void __iomem *addr;
u16 offset = 0;
u32 cap_hdr;

if (WARN_ON_ONCE(rcrb == CXL_RESOURCE_NONE))
return 0;

if (!request_mem_region(rcrb, SZ_4K, dev_name(dev)))
return 0;

addr = ioremap(rcrb, SZ_4K);
if (!addr)
goto out;

cap_hdr = readl(addr + offset);
while (PCI_EXT_CAP_ID(cap_hdr) != PCI_EXT_CAP_ID_ERR) {
offset = PCI_EXT_CAP_NEXT(cap_hdr);

/* Offset 0 terminates capability list. */
if (!offset)
break;
cap_hdr = readl(addr + offset);
}

if (offset)
dev_dbg(dev, "found AER extended capability (0x%x)\n", offset);

iounmap(addr);
out:
release_mem_region(rcrb, SZ_4K);

return offset;
}

resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri,
enum cxl_rcrb which)
{
Expand Down
7 changes: 7 additions & 0 deletions drivers/cxl/cxl.h
Original file line number Diff line number Diff line change
Expand Up @@ -704,6 +704,13 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
struct device *dport_dev, int port_id,
resource_size_t rcrb);

#ifdef CONFIG_PCIEAER_CXL
void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport);
#else
static inline void cxl_setup_parent_dport(struct device *host,
struct cxl_dport *dport) { }
#endif

struct cxl_decoder *to_cxl_decoder(struct device *dev);
struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev);
struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev);
Expand Down
2 changes: 2 additions & 0 deletions drivers/cxl/mem.c
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,8 @@ static int cxl_mem_probe(struct device *dev)
else
endpoint_parent = &parent_port->dev;

cxl_setup_parent_dport(dev, dport);

device_lock(endpoint_parent);
if (!endpoint_parent->driver) {
dev_err(dev, "CXL port topology %s not enabled\n",
Expand Down

0 comments on commit f05fd10

Please sign in to comment.