Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-23 12:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity
d27f3a348e
linux/drivers/edac/amd64_edac.c
Yazen Ghannam d27f3a348e EDAC, amd64: Determine EDAC capabilities on Fam17h systems 
We need to determine the EDAC capabilities from all UMCs on the node. We
should only check UMCs that are enabled and make sure they all agree.

Signed-off-by: Yazen Ghannam <Yazen.Ghannam@amd.com>
Cc: Aravind Gopalakrishnan <aravindksg.lkml@gmail.com>
Cc: linux-edac <linux-edac@vger.kernel.org>
Cc: x86-ml <x86@kernel.org>
Link: http://lkml.kernel.org/r/1479423463-8536-15-git-send-email-Yazen.Ghannam@amd.com
Signed-off-by: Borislav Petkov <bp@suse.de>
2016-11-29 18:05:47 +01:00

3448 lines
87 KiB
C
Raw Blame History

#include "amd64_edac.h"
#include <asm/amd_nb.h>
static struct edac_pci_ctl_info *pci_ctl;
static int report_gart_errors;
module_param(report_gart_errors, int, 0644);
/*
* Set by command line parameter. If BIOS has enabled the ECC, this override is
* cleared to prevent re-enabling the hardware by this driver.
*/
static int ecc_enable_override;
module_param(ecc_enable_override, int, 0644);
static struct msr __percpu *msrs;
/* Per-node stuff */
static struct ecc_settings **ecc_stngs;
/*
* Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
* bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
* or higher value'.
*
*FIXME: Produce a better mapping/linearisation.
*/
static const struct scrubrate {
u32 scrubval; /* bit pattern for scrub rate */
u32 bandwidth; /* bandwidth consumed (bytes/sec) */
} scrubrates[] = {
{ 0x01, 1600000000UL},
{ 0x02, 800000000UL},
{ 0x03, 400000000UL},
{ 0x04, 200000000UL},
{ 0x05, 100000000UL},
{ 0x06, 50000000UL},
{ 0x07, 25000000UL},
{ 0x08, 12284069UL},
{ 0x09, 6274509UL},
{ 0x0A, 3121951UL},
{ 0x0B, 1560975UL},
{ 0x0C, 781440UL},
{ 0x0D, 390720UL},
{ 0x0E, 195300UL},
{ 0x0F, 97650UL},
{ 0x10, 48854UL},
{ 0x11, 24427UL},
{ 0x12, 12213UL},
{ 0x13, 6101UL},
{ 0x14, 3051UL},
{ 0x15, 1523UL},
{ 0x16, 761UL},
{ 0x00, 0UL}, /* scrubbing off */
};
int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
u32 *val, const char *func)
{
int err = 0;
err = pci_read_config_dword(pdev, offset, val);
if (err)
amd64_warn("%s: error reading F%dx%03x.\n",
func, PCI_FUNC(pdev->devfn), offset);
return err;
}
int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
u32 val, const char *func)
{
int err = 0;
err = pci_write_config_dword(pdev, offset, val);
if (err)
amd64_warn("%s: error writing to F%dx%03x.\n",
func, PCI_FUNC(pdev->devfn), offset);
return err;
}
/*
* Select DCT to which PCI cfg accesses are routed
*/
static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
{
u32 reg = 0;
amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
reg &= (pvt->model == 0x30) ? ~3 : ~1;
reg |= dct;
amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
}
/*
*
* Depending on the family, F2 DCT reads need special handling:
*
* K8: has a single DCT only and no address offsets >= 0x100
*
* F10h: each DCT has its own set of regs
* DCT0 -> F2x040..
* DCT1 -> F2x140..
*
* F16h: has only 1 DCT
*
* F15h: we select which DCT we access using F1x10C[DctCfgSel]
*/
static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
int offset, u32 *val)
{
switch (pvt->fam) {
case 0xf:
if (dct || offset >= 0x100)
return -EINVAL;
break;
case 0x10:
if (dct) {
/*
* Note: If ganging is enabled, barring the regs
* F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
* return 0. (cf. Section 2.8.1 F10h BKDG)
*/
if (dct_ganging_enabled(pvt))
return 0;
offset += 0x100;
}
break;
case 0x15:
/*
* F15h: F2x1xx addresses do not map explicitly to DCT1.
* We should select which DCT we access using F1x10C[DctCfgSel]
*/
dct = (dct && pvt->model == 0x30) ? 3 : dct;
f15h_select_dct(pvt, dct);
break;
case 0x16:
if (dct)
return -EINVAL;
break;
default:
break;
}
return amd64_read_pci_cfg(pvt->F2, offset, val);
}
/*
* Memory scrubber control interface. For K8, memory scrubbing is handled by
* hardware and can involve L2 cache, dcache as well as the main memory. With
* F10, this is extended to L3 cache scrubbing on CPU models sporting that
* functionality.
*
* This causes the "units" for the scrubbing speed to vary from 64 byte blocks
* (dram) over to cache lines. This is nasty, so we will use bandwidth in
* bytes/sec for the setting.
*
* Currently, we only do dram scrubbing. If the scrubbing is done in software on
* other archs, we might not have access to the caches directly.
*/
static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
{
/*
* Fam17h supports scrub values between 0x5 and 0x14. Also, the values
* are shifted down by 0x5, so scrubval 0x5 is written to the register
* as 0x0, scrubval 0x6 as 0x1, etc.
*/
if (scrubval >= 0x5 && scrubval <= 0x14) {
scrubval -= 0x5;
pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
} else {
pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
}
}
/*
* Scan the scrub rate mapping table for a close or matching bandwidth value to
* issue. If requested is too big, then use last maximum value found.
*/
static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
{
u32 scrubval;
int i;
/*
* map the configured rate (new_bw) to a value specific to the AMD64
* memory controller and apply to register. Search for the first
* bandwidth entry that is greater or equal than the setting requested
* and program that. If at last entry, turn off DRAM scrubbing.
*
* If no suitable bandwidth is found, turn off DRAM scrubbing entirely
* by falling back to the last element in scrubrates[].
*/
for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
/*
* skip scrub rates which aren't recommended
* (see F10 BKDG, F3x58)
*/
if (scrubrates[i].scrubval < min_rate)
continue;
if (scrubrates[i].bandwidth <= new_bw)
break;
}
scrubval = scrubrates[i].scrubval;
if (pvt->fam == 0x17) {
__f17h_set_scrubval(pvt, scrubval);
} else if (pvt->fam == 0x15 && pvt->model == 0x60) {
f15h_select_dct(pvt, 0);
pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
f15h_select_dct(pvt, 1);
pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
} else {
pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
}
if (scrubval)
return scrubrates[i].bandwidth;
return 0;
}
static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
{
struct amd64_pvt *pvt = mci->pvt_info;
u32 min_scrubrate = 0x5;
if (pvt->fam == 0xf)
min_scrubrate = 0x0;
if (pvt->fam == 0x15) {
/* Erratum #505 */
if (pvt->model < 0x10)
f15h_select_dct(pvt, 0);
if (pvt->model == 0x60)
min_scrubrate = 0x6;
}
return __set_scrub_rate(pvt, bw, min_scrubrate);
}
static int get_scrub_rate(struct mem_ctl_info *mci)
{
struct amd64_pvt *pvt = mci->pvt_info;
int i, retval = -EINVAL;
u32 scrubval = 0;
switch (pvt->fam) {
case 0x15:
/* Erratum #505 */
if (pvt->model < 0x10)
f15h_select_dct(pvt, 0);
if (pvt->model == 0x60)
amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
break;
case 0x17:
amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
if (scrubval & BIT(0)) {
amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
scrubval &= 0xF;
scrubval += 0x5;
} else {
scrubval = 0;
}
break;
default:
amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
break;
}
scrubval = scrubval & 0x001F;
for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
if (scrubrates[i].scrubval == scrubval) {
retval = scrubrates[i].bandwidth;
break;
}
}
return retval;
}
/*
* returns true if the SysAddr given by sys_addr matches the
* DRAM base/limit associated with node_id
*/
static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
{
u64 addr;
/* The K8 treats this as a 40-bit value. However, bits 63-40 will be
* all ones if the most significant implemented address bit is 1.
* Here we discard bits 63-40. See section 3.4.2 of AMD publication
* 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
* Application Programming.
*/
addr = sys_addr & 0x000000ffffffffffull;
return ((addr >= get_dram_base(pvt, nid)) &&
(addr <= get_dram_limit(pvt, nid)));
}
/*
* Attempt to map a SysAddr to a node. On success, return a pointer to the
* mem_ctl_info structure for the node that the SysAddr maps to.
*
* On failure, return NULL.
*/
static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
u64 sys_addr)
{
struct amd64_pvt *pvt;
u8 node_id;
u32 intlv_en, bits;
/*
* Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
* 3.4.4.2) registers to map the SysAddr to a node ID.
*/
pvt = mci->pvt_info;
/*
* The value of this field should be the same for all DRAM Base
* registers. Therefore we arbitrarily choose to read it from the
* register for node 0.
*/
intlv_en = dram_intlv_en(pvt, 0);
if (intlv_en == 0) {
for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
if (base_limit_match(pvt, sys_addr, node_id))
goto found;
}
goto err_no_match;
}
if (unlikely((intlv_en != 0x01) &&
(intlv_en != 0x03) &&
(intlv_en != 0x07))) {
amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
return NULL;
}
bits = (((u32) sys_addr) >> 12) & intlv_en;
for (node_id = 0; ; ) {
if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
break; /* intlv_sel field matches */
if (++node_id >= DRAM_RANGES)
goto err_no_match;
}
/* sanity test for sys_addr */
if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
"range for node %d with node interleaving enabled.\n",
__func__, sys_addr, node_id);
return NULL;
}
found:
return edac_mc_find((int)node_id);
err_no_match:
edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
(unsigned long)sys_addr);
return NULL;
}
/*
* compute the CS base address of the @csrow on the DRAM controller @dct.
* For details see F2x[5C:40] in the processor's BKDG
*/
static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
u64 *base, u64 *mask)
{
u64 csbase, csmask, base_bits, mask_bits;
u8 addr_shift;
if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
csbase = pvt->csels[dct].csbases[csrow];
csmask = pvt->csels[dct].csmasks[csrow];
base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
addr_shift = 4;
/*
* F16h and F15h, models 30h and later need two addr_shift values:
* 8 for high and 6 for low (cf. F16h BKDG).
*/
} else if (pvt->fam == 0x16 ||
(pvt->fam == 0x15 && pvt->model >= 0x30)) {
csbase = pvt->csels[dct].csbases[csrow];
csmask = pvt->csels[dct].csmasks[csrow >> 1];
*base = (csbase & GENMASK_ULL(15, 5)) << 6;
*base |= (csbase & GENMASK_ULL(30, 19)) << 8;
*mask = ~0ULL;
/* poke holes for the csmask */
*mask &= ~((GENMASK_ULL(15, 5) << 6) |
(GENMASK_ULL(30, 19) << 8));
*mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
*mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
return;
} else {
csbase = pvt->csels[dct].csbases[csrow];
csmask = pvt->csels[dct].csmasks[csrow >> 1];
addr_shift = 8;
if (pvt->fam == 0x15)
base_bits = mask_bits =
GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
else
base_bits = mask_bits =
GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
}
*base = (csbase & base_bits) << addr_shift;
*mask = ~0ULL;
/* poke holes for the csmask */
*mask &= ~(mask_bits << addr_shift);
/* OR them in */
*mask |= (csmask & mask_bits) << addr_shift;
}
#define for_each_chip_select(i, dct, pvt) \
for (i = 0; i < pvt->csels[dct].b_cnt; i++)
#define chip_select_base(i, dct, pvt) \
pvt->csels[dct].csbases[i]
#define for_each_chip_select_mask(i, dct, pvt) \
for (i = 0; i < pvt->csels[dct].m_cnt; i++)
/*
* @input_addr is an InputAddr associated with the node given by mci. Return the
* csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
*/
static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
{
struct amd64_pvt *pvt;
int csrow;
u64 base, mask;
pvt = mci->pvt_info;
for_each_chip_select(csrow, 0, pvt) {
if (!csrow_enabled(csrow, 0, pvt))
continue;
get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
mask = ~mask;
if ((input_addr & mask) == (base & mask)) {
edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
(unsigned long)input_addr, csrow,
pvt->mc_node_id);
return csrow;
}
}
edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
(unsigned long)input_addr, pvt->mc_node_id);
return -1;
}
/*
* Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
* for the node represented by mci. Info is passed back in *hole_base,
* *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
* info is invalid. Info may be invalid for either of the following reasons:
*
* - The revision of the node is not E or greater. In this case, the DRAM Hole
* Address Register does not exist.
*
* - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
* indicating that its contents are not valid.
*
* The values passed back in *hole_base, *hole_offset, and *hole_size are
* complete 32-bit values despite the fact that the bitfields in the DHAR
* only represent bits 31-24 of the base and offset values.
*/
int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
u64 *hole_offset, u64 *hole_size)
{
struct amd64_pvt *pvt = mci->pvt_info;
/* only revE and later have the DRAM Hole Address Register */
if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
edac_dbg(1, " revision %d for node %d does not support DHAR\n",
pvt->ext_model, pvt->mc_node_id);
return 1;
}
/* valid for Fam10h and above */
if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
return 1;
}
if (!dhar_valid(pvt)) {
edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
pvt->mc_node_id);
return 1;
}
/* This node has Memory Hoisting */
/* +------------------+--------------------+--------------------+-----
* | memory | DRAM hole | relocated |
* | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
* | | | DRAM hole |
* | | | [0x100000000, |
* | | | (0x100000000+ |
* | | | (0xffffffff-x))] |
* +------------------+--------------------+--------------------+-----
*
* Above is a diagram of physical memory showing the DRAM hole and the
* relocated addresses from the DRAM hole. As shown, the DRAM hole
* starts at address x (the base address) and extends through address
* 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
* addresses in the hole so that they start at 0x100000000.
*/
*hole_base = dhar_base(pvt);
*hole_size = (1ULL << 32) - *hole_base;
*hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
: k8_dhar_offset(pvt);
edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
pvt->mc_node_id, (unsigned long)*hole_base,
(unsigned long)*hole_offset, (unsigned long)*hole_size);
return 0;
}
EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
/*
* Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
* assumed that sys_addr maps to the node given by mci.
*
* The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
* 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
* SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
* then it is also involved in translating a SysAddr to a DramAddr. Sections
* 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
* These parts of the documentation are unclear. I interpret them as follows:
*
* When node n receives a SysAddr, it processes the SysAddr as follows:
*
* 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
* Limit registers for node n. If the SysAddr is not within the range
* specified by the base and limit values, then node n ignores the Sysaddr
* (since it does not map to node n). Otherwise continue to step 2 below.
*
* 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
* disabled so skip to step 3 below. Otherwise see if the SysAddr is within
* the range of relocated addresses (starting at 0x100000000) from the DRAM
* hole. If not, skip to step 3 below. Else get the value of the
* DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
* offset defined by this value from the SysAddr.
*
* 3. Obtain the base address for node n from the DRAMBase field of the DRAM
* Base register for node n. To obtain the DramAddr, subtract the base
* address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
*/
static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
{
struct amd64_pvt *pvt = mci->pvt_info;
u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
int ret;
dram_base = get_dram_base(pvt, pvt->mc_node_id);
ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
&hole_size);
if (!ret) {
if ((sys_addr >= (1ULL << 32)) &&
(sys_addr < ((1ULL << 32) + hole_size))) {
/* use DHAR to translate SysAddr to DramAddr */
dram_addr = sys_addr - hole_offset;
edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
(unsigned long)sys_addr,
(unsigned long)dram_addr);
return dram_addr;
}
}
/*
* Translate the SysAddr to a DramAddr as shown near the start of
* section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
* only deals with 40-bit values. Therefore we discard bits 63-40 of
* sys_addr below. If bit 39 of sys_addr is 1 then the bits we
* discard are all 1s. Otherwise the bits we discard are all 0s. See
* section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
* Programmer's Manual Volume 1 Application Programming.
*/
dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
(unsigned long)sys_addr, (unsigned long)dram_addr);
return dram_addr;
}
/*
* @intlv_en is the value of the IntlvEn field from a DRAM Base register
* (section 3.4.4.1). Return the number of bits from a SysAddr that are used
* for node interleaving.
*/
static int num_node_interleave_bits(unsigned intlv_en)
{
static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
int n;
BUG_ON(intlv_en > 7);
n = intlv_shift_table[intlv_en];
return n;
}
/* Translate the DramAddr given by @dram_addr to an InputAddr. */
static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
{
struct amd64_pvt *pvt;
int intlv_shift;
u64 input_addr;
pvt = mci->pvt_info;
/*
* See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
* concerning translating a DramAddr to an InputAddr.
*/
intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
(dram_addr & 0xfff);
edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
intlv_shift, (unsigned long)dram_addr,
(unsigned long)input_addr);
return input_addr;
}
/*
* Translate the SysAddr represented by @sys_addr to an InputAddr. It is
* assumed that @sys_addr maps to the node given by mci.
*/
static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
{
u64 input_addr;
input_addr =
dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
(unsigned long)sys_addr, (unsigned long)input_addr);
return input_addr;
}
/* Map the Error address to a PAGE and PAGE OFFSET. */
static inline void error_address_to_page_and_offset(u64 error_address,
struct err_info *err)
{
err->page = (u32) (error_address >> PAGE_SHIFT);
err->offset = ((u32) error_address) & ~PAGE_MASK;
}
/*
* @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
* Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
* of a node that detected an ECC memory error. mci represents the node that
* the error address maps to (possibly different from the node that detected
* the error). Return the number of the csrow that sys_addr maps to, or -1 on
* error.
*/
static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
{
int csrow;
csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
if (csrow == -1)
amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
"address 0x%lx\n", (unsigned long)sys_addr);
return csrow;
}
static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
/*
* Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
* are ECC capable.
*/
static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
{
unsigned long edac_cap = EDAC_FLAG_NONE;
u8 bit;
if (pvt->umc) {
u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
for (i = 0; i < NUM_UMCS; i++) {
if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
continue;
umc_en_mask |= BIT(i);
/* UMC Configuration bit 12 (DimmEccEn) */
if (pvt->umc[i].umc_cfg & BIT(12))
dimm_ecc_en_mask |= BIT(i);
}
if (umc_en_mask == dimm_ecc_en_mask)
edac_cap = EDAC_FLAG_SECDED;
} else {
bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
? 19
: 17;
if (pvt->dclr0 & BIT(bit))
edac_cap = EDAC_FLAG_SECDED;
}
return edac_cap;
}
static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
{
edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
if (pvt->dram_type == MEM_LRDDR3) {
u32 dcsm = pvt->csels[chan].csmasks[0];
/*
* It's assumed all LRDIMMs in a DCT are going to be of
* same 'type' until proven otherwise. So, use a cs
* value of '0' here to get dcsm value.
*/
edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
}
edac_dbg(1, "All DIMMs support ECC:%s\n",
(dclr & BIT(19)) ? "yes" : "no");
edac_dbg(1, " PAR/ERR parity: %s\n",
(dclr & BIT(8)) ? "enabled" : "disabled");
if (pvt->fam == 0x10)
edac_dbg(1, " DCT 128bit mode width: %s\n",
(dclr & BIT(11)) ? "128b" : "64b");
edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
(dclr & BIT(12)) ? "yes" : "no",
(dclr & BIT(13)) ? "yes" : "no",
(dclr & BIT(14)) ? "yes" : "no",
(dclr & BIT(15)) ? "yes" : "no");
}
static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
{
u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
int dimm, size0, size1;
edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
for (dimm = 0; dimm < 4; dimm++) {
size0 = 0;
if (dcsb[dimm*2] & DCSB_CS_ENABLE)
size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, dimm);
size1 = 0;
if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, dimm);
amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
dimm * 2, size0,
dimm * 2 + 1, size1);
}
}
static void __dump_misc_regs_df(struct amd64_pvt *pvt)
{
struct amd64_umc *umc;
u32 i, tmp, umc_base;
for (i = 0; i < NUM_UMCS; i++) {
umc_base = get_umc_base(i);
umc = &pvt->umc[i];
edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
(umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
if (pvt->dram_type == MEM_LRDDR4) {
amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
i, 1 << ((tmp >> 4) & 0x3));
}
debug_display_dimm_sizes_df(pvt, i);
}
edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
pvt->dhar, dhar_base(pvt));
}
/* Display and decode various NB registers for debug purposes. */
static void __dump_misc_regs(struct amd64_pvt *pvt)
{
edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
edac_dbg(1, " NB two channel DRAM capable: %s\n",
(pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
(pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
(pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
pvt->dhar, dhar_base(pvt),
(pvt->fam == 0xf) ? k8_dhar_offset(pvt)
: f10_dhar_offset(pvt));
debug_display_dimm_sizes(pvt, 0);
/* everything below this point is Fam10h and above */
if (pvt->fam == 0xf)
return;
debug_display_dimm_sizes(pvt, 1);
/* Only if NOT ganged does dclr1 have valid info */
if (!dct_ganging_enabled(pvt))
debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
}
/* Display and decode various NB registers for debug purposes. */
static void dump_misc_regs(struct amd64_pvt *pvt)
{
if (pvt->umc)
__dump_misc_regs_df(pvt);
else
__dump_misc_regs(pvt);
edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
amd64_info("using %s syndromes.\n",
((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
}
/*
* See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
*/
static void prep_chip_selects(struct amd64_pvt *pvt)
{
if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
} else if (pvt->fam == 0x15 && pvt->model == 0x30) {
pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
} else {
pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
}
}
/*
* Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
*/
static void read_dct_base_mask(struct amd64_pvt *pvt)
{
int base_reg0, base_reg1, mask_reg0, mask_reg1, cs;
prep_chip_selects(pvt);
if (pvt->umc) {
base_reg0 = get_umc_base(0) + UMCCH_BASE_ADDR;
base_reg1 = get_umc_base(1) + UMCCH_BASE_ADDR;
mask_reg0 = get_umc_base(0) + UMCCH_ADDR_MASK;
mask_reg1 = get_umc_base(1) + UMCCH_ADDR_MASK;
} else {
base_reg0 = DCSB0;
base_reg1 = DCSB1;
mask_reg0 = DCSM0;
mask_reg1 = DCSM1;
}
for_each_chip_select(cs, 0, pvt) {
int reg0 = base_reg0 + (cs * 4);
int reg1 = base_reg1 + (cs * 4);
u32 *base0 = &pvt->csels[0].csbases[cs];
u32 *base1 = &pvt->csels[1].csbases[cs];
if (pvt->umc) {
if (!amd_smn_read(pvt->mc_node_id, reg0, base0))
edac_dbg(0, " DCSB0[%d]=0x%08x reg: 0x%x\n",
cs, *base0, reg0);
if (!amd_smn_read(pvt->mc_node_id, reg1, base1))
edac_dbg(0, " DCSB1[%d]=0x%08x reg: 0x%x\n",
cs, *base1, reg1);
} else {
if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
cs, *base0, reg0);
if (pvt->fam == 0xf)
continue;
if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
cs, *base1, (pvt->fam == 0x10) ? reg1
: reg0);
}
}
for_each_chip_select_mask(cs, 0, pvt) {
int reg0 = mask_reg0 + (cs * 4);
int reg1 = mask_reg1 + (cs * 4);
u32 *mask0 = &pvt->csels[0].csmasks[cs];
u32 *mask1 = &pvt->csels[1].csmasks[cs];
if (pvt->umc) {
if (!amd_smn_read(pvt->mc_node_id, reg0, mask0))
edac_dbg(0, " DCSM0[%d]=0x%08x reg: 0x%x\n",
cs, *mask0, reg0);
if (!amd_smn_read(pvt->mc_node_id, reg1, mask1))
edac_dbg(0, " DCSM1[%d]=0x%08x reg: 0x%x\n",
cs, *mask1, reg1);
} else {
if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
cs, *mask0, reg0);
if (pvt->fam == 0xf)
continue;
if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
cs, *mask1, (pvt->fam == 0x10) ? reg1
: reg0);
}
}
}
static void determine_memory_type(struct amd64_pvt *pvt)
{
u32 dram_ctrl, dcsm;
switch (pvt->fam) {
case 0xf:
if (pvt->ext_model >= K8_REV_F)
goto ddr3;
pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
return;
case 0x10:
if (pvt->dchr0 & DDR3_MODE)
goto ddr3;
pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
return;
case 0x15:
if (pvt->model < 0x60)
goto ddr3;
/*
* Model 0x60h needs special handling:
*
* We use a Chip Select value of '0' to obtain dcsm.
* Theoretically, it is possible to populate LRDIMMs of different
* 'Rank' value on a DCT. But this is not the common case. So,
* it's reasonable to assume all DIMMs are going to be of same
* 'type' until proven otherwise.
*/
amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
dcsm = pvt->csels[0].csmasks[0];
if (((dram_ctrl >> 8) & 0x7) == 0x2)
pvt->dram_type = MEM_DDR4;
else if (pvt->dclr0 & BIT(16))
pvt->dram_type = MEM_DDR3;
else if (dcsm & 0x3)
pvt->dram_type = MEM_LRDDR3;
else
pvt->dram_type = MEM_RDDR3;
return;
case 0x16:
goto ddr3;
case 0x17:
if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
pvt->dram_type = MEM_LRDDR4;
else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
pvt->dram_type = MEM_RDDR4;
else
pvt->dram_type = MEM_DDR4;
return;
default:
WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
pvt->dram_type = MEM_EMPTY;
}
return;
ddr3:
pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
}
/* Get the number of DCT channels the memory controller is using. */
static int k8_early_channel_count(struct amd64_pvt *pvt)
{
int flag;
if (pvt->ext_model >= K8_REV_F)
/* RevF (NPT) and later */
flag = pvt->dclr0 & WIDTH_128;
else
/* RevE and earlier */
flag = pvt->dclr0 & REVE_WIDTH_128;
/* not used */
pvt->dclr1 = 0;
return (flag) ? 2 : 1;
}
/* On F10h and later ErrAddr is MC4_ADDR[47:1] */
static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
{
u16 mce_nid = amd_get_nb_id(m->extcpu);
struct mem_ctl_info *mci;
u8 start_bit = 1;
u8 end_bit = 47;
u64 addr;
mci = edac_mc_find(mce_nid);
if (!mci)
return 0;
pvt = mci->pvt_info;
if (pvt->fam == 0xf) {
start_bit = 3;
end_bit = 39;
}
addr = m->addr & GENMASK_ULL(end_bit, start_bit);
/*
* Erratum 637 workaround
*/
if (pvt->fam == 0x15) {
u64 cc6_base, tmp_addr;
u32 tmp;
u8 intlv_en;
if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
return addr;
amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
intlv_en = tmp >> 21 & 0x7;
/* add [47:27] + 3 trailing bits */
cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
/* reverse and add DramIntlvEn */
cc6_base |= intlv_en ^ 0x7;
/* pin at [47:24] */
cc6_base <<= 24;
if (!intlv_en)
return cc6_base | (addr & GENMASK_ULL(23, 0));
amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
/* faster log2 */
tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
/* OR DramIntlvSel into bits [14:12] */
tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
/* add remaining [11:0] bits from original MC4_ADDR */
tmp_addr |= addr & GENMASK_ULL(11, 0);
return cc6_base | tmp_addr;
}
return addr;
}
static struct pci_dev *pci_get_related_function(unsigned int vendor,
unsigned int device,
struct pci_dev *related)
{
struct pci_dev *dev = NULL;
while ((dev = pci_get_device(vendor, device, dev))) {
if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
(dev->bus->number == related->bus->number) &&
(PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
break;
}
return dev;
}
static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
{
struct amd_northbridge *nb;
struct pci_dev *f1 = NULL;
unsigned int pci_func;
int off = range << 3;
u32 llim;
amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
if (pvt->fam == 0xf)
return;
if (!dram_rw(pvt, range))
return;
amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
/* F15h: factor in CC6 save area by reading dst node's limit reg */
if (pvt->fam != 0x15)
return;
nb = node_to_amd_nb(dram_dst_node(pvt, range));
if (WARN_ON(!nb))
return;
if (pvt->model == 0x60)
pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
else if (pvt->model == 0x30)
pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
else
pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
if (WARN_ON(!f1))
return;
amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
/* {[39:27],111b} */
pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
/* [47:40] */
pvt->ranges[range].lim.hi |= llim >> 13;
pci_dev_put(f1);
}
static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
struct err_info *err)
{
struct amd64_pvt *pvt = mci->pvt_info;
error_address_to_page_and_offset(sys_addr, err);
/*
* Find out which node the error address belongs to. This may be
* different from the node that detected the error.
*/
err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
if (!err->src_mci) {
amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
(unsigned long)sys_addr);
err->err_code = ERR_NODE;
return;
}
/* Now map the sys_addr to a CSROW */
err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
if (err->csrow < 0) {
err->err_code = ERR_CSROW;
return;
}
/* CHIPKILL enabled */
if (pvt->nbcfg & NBCFG_CHIPKILL) {
err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
if (err->channel < 0) {
/*
* Syndrome didn't map, so we don't know which of the
* 2 DIMMs is in error. So we need to ID 'both' of them
* as suspect.
*/
amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
"possible error reporting race\n",
err->syndrome);
err->err_code = ERR_CHANNEL;
return;
}
} else {
/*
* non-chipkill ecc mode
*
* The k8 documentation is unclear about how to determine the
* channel number when using non-chipkill memory. This method
* was obtained from email communication with someone at AMD.
* (Wish the email was placed in this comment - norsk)
*/
err->channel = ((sys_addr & BIT(3)) != 0);
}
}
static int ddr2_cs_size(unsigned i, bool dct_width)
{
unsigned shift = 0;
if (i <= 2)
shift = i;
else if (!(i & 0x1))
shift = i >> 1;
else
shift = (i + 1) >> 1;
return 128 << (shift + !!dct_width);
}
static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
unsigned cs_mode, int cs_mask_nr)
{
u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
if (pvt->ext_model >= K8_REV_F) {
WARN_ON(cs_mode > 11);
return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
}
else if (pvt->ext_model >= K8_REV_D) {
unsigned diff;
WARN_ON(cs_mode > 10);
/*
* the below calculation, besides trying to win an obfuscated C
* contest, maps cs_mode values to DIMM chip select sizes. The
* mappings are:
*
* cs_mode CS size (mb)
* ======= ============
* 0 32
* 1 64
* 2 128
* 3 128
* 4 256
* 5 512
* 6 256
* 7 512
* 8 1024
* 9 1024
* 10 2048
*
* Basically, it calculates a value with which to shift the
* smallest CS size of 32MB.
*
* ddr[23]_cs_size have a similar purpose.
*/
diff = cs_mode/3 + (unsigned)(cs_mode > 5);
return 32 << (cs_mode - diff);
}
else {
WARN_ON(cs_mode > 6);
return 32 << cs_mode;
}
}
/*
* Get the number of DCT channels in use.
*
* Return:
* number of Memory Channels in operation
* Pass back:
* contents of the DCL0_LOW register
*/
static int f1x_early_channel_count(struct amd64_pvt *pvt)
{
int i, j, channels = 0;
/* On F10h, if we are in 128 bit mode, then we are using 2 channels */
if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
return 2;
/*
* Need to check if in unganged mode: In such, there are 2 channels,
* but they are not in 128 bit mode and thus the above 'dclr0' status
* bit will be OFF.
*
* Need to check DCT0[0] and DCT1[0] to see if only one of them has
* their CSEnable bit on. If so, then SINGLE DIMM case.
*/
edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
/*
* Check DRAM Bank Address Mapping values for each DIMM to see if there
* is more than just one DIMM present in unganged mode. Need to check
* both controllers since DIMMs can be placed in either one.
*/
for (i = 0; i < 2; i++) {
u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
for (j = 0; j < 4; j++) {
if (DBAM_DIMM(j, dbam) > 0) {
channels++;
break;
}
}
}
if (channels > 2)
channels = 2;
amd64_info("MCT channel count: %d\n", channels);
return channels;
}
static int f17_early_channel_count(struct amd64_pvt *pvt)
{
int i, channels = 0;
/* SDP Control bit 31 (SdpInit) is clear for unused UMC channels */
for (i = 0; i < NUM_UMCS; i++)
channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
amd64_info("MCT channel count: %d\n", channels);
return channels;
}
static int ddr3_cs_size(unsigned i, bool dct_width)
{
unsigned shift = 0;
int cs_size = 0;
if (i == 0 || i == 3 || i == 4)
cs_size = -1;
else if (i <= 2)
shift = i;
else if (i == 12)
shift = 7;
else if (!(i & 0x1))
shift = i >> 1;
else
shift = (i + 1) >> 1;
if (cs_size != -1)
cs_size = (128 * (1 << !!dct_width)) << shift;
return cs_size;
}
static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
{
unsigned shift = 0;
int cs_size = 0;
if (i < 4 || i == 6)
cs_size = -1;
else if (i == 12)
shift = 7;
else if (!(i & 0x1))
shift = i >> 1;
else
shift = (i + 1) >> 1;
if (cs_size != -1)
cs_size = rank_multiply * (128 << shift);
return cs_size;
}
static int ddr4_cs_size(unsigned i)
{
int cs_size = 0;
if (i == 0)
cs_size = -1;
else if (i == 1)
cs_size = 1024;
else
/* Min cs_size = 1G */
cs_size = 1024 * (1 << (i >> 1));
return cs_size;
}
static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
unsigned cs_mode, int cs_mask_nr)
{
u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
WARN_ON(cs_mode > 11);
if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
else
return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
}
/*
* F15h supports only 64bit DCT interfaces
*/
static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
unsigned cs_mode, int cs_mask_nr)
{
WARN_ON(cs_mode > 12);
return ddr3_cs_size(cs_mode, false);
}
/* F15h M60h supports DDR4 mapping as well.. */
static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
unsigned cs_mode, int cs_mask_nr)
{
int cs_size;
u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
WARN_ON(cs_mode > 12);
if (pvt->dram_type == MEM_DDR4) {
if (cs_mode > 9)
return -1;
cs_size = ddr4_cs_size(cs_mode);
} else if (pvt->dram_type == MEM_LRDDR3) {
unsigned rank_multiply = dcsm & 0xf;
if (rank_multiply == 3)
rank_multiply = 4;
cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
} else {
/* Minimum cs size is 512mb for F15hM60h*/
if (cs_mode == 0x1)
return -1;
cs_size = ddr3_cs_size(cs_mode, false);
}
return cs_size;
}
/*
* F16h and F15h model 30h have only limited cs_modes.
*/
static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
unsigned cs_mode, int cs_mask_nr)
{
WARN_ON(cs_mode > 12);
if (cs_mode == 6 || cs_mode == 8 ||
cs_mode == 9 || cs_mode == 12)
return -1;
else
return ddr3_cs_size(cs_mode, false);
}
static int f17_base_addr_to_cs_size(struct amd64_pvt *pvt, u8 umc,
unsigned int cs_mode, int csrow_nr)
{
u32 base_addr = pvt->csels[umc].csbases[csrow_nr];
/* Each mask is used for every two base addresses. */
u32 addr_mask = pvt->csels[umc].csmasks[csrow_nr >> 1];
/* Register [31:1] = Address [39:9]. Size is in kBs here. */
u32 size = ((addr_mask >> 1) - (base_addr >> 1) + 1) >> 1;
edac_dbg(1, "BaseAddr: 0x%x, AddrMask: 0x%x\n", base_addr, addr_mask);
/* Return size in MBs. */
return size >> 10;
}
static void read_dram_ctl_register(struct amd64_pvt *pvt)
{
if (pvt->fam == 0xf)
return;
if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
edac_dbg(0, " DCTs operate in %s mode\n",
(dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
if (!dct_ganging_enabled(pvt))
edac_dbg(0, " Address range split per DCT: %s\n",
(dct_high_range_enabled(pvt) ? "yes" : "no"));
edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
(dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
(dct_memory_cleared(pvt) ? "yes" : "no"));
edac_dbg(0, " channel interleave: %s, "
"interleave bits selector: 0x%x\n",
(dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
dct_sel_interleave_addr(pvt));
}
amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
}
/*
* Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
* 2.10.12 Memory Interleaving Modes).
*/
static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
u8 intlv_en, int num_dcts_intlv,
u32 dct_sel)
{
u8 channel = 0;
u8 select;
if (!(intlv_en))
return (u8)(dct_sel);
if (num_dcts_intlv == 2) {
select = (sys_addr >> 8) & 0x3;
channel = select ? 0x3 : 0;
} else if (num_dcts_intlv == 4) {
u8 intlv_addr = dct_sel_interleave_addr(pvt);
switch (intlv_addr) {
case 0x4:
channel = (sys_addr >> 8) & 0x3;
break;
case 0x5:
channel = (sys_addr >> 9) & 0x3;
break;
}
}
return channel;
}
/*
* Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
* Interleaving Modes.
*/
static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
bool hi_range_sel, u8 intlv_en)
{
u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
if (dct_ganging_enabled(pvt))
return 0;
if (hi_range_sel)
return dct_sel_high;
/*
* see F2x110[DctSelIntLvAddr] - channel interleave mode
*/
if (dct_interleave_enabled(pvt)) {
u8 intlv_addr = dct_sel_interleave_addr(pvt);
/* return DCT select function: 0=DCT0, 1=DCT1 */
if (!intlv_addr)
return sys_addr >> 6 & 1;
if (intlv_addr & 0x2) {
u8 shift = intlv_addr & 0x1 ? 9 : 6;
u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
return ((sys_addr >> shift) & 1) ^ temp;
}
if (intlv_addr & 0x4) {
u8 shift = intlv_addr & 0x1 ? 9 : 8;
return (sys_addr >> shift) & 1;
}
return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
}
if (dct_high_range_enabled(pvt))
return ~dct_sel_high & 1;
return 0;
}
/* Convert the sys_addr to the normalized DCT address */
static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
u64 sys_addr, bool hi_rng,
u32 dct_sel_base_addr)
{
u64 chan_off;
u64 dram_base = get_dram_base(pvt, range);
u64 hole_off = f10_dhar_offset(pvt);
u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
if (hi_rng) {
/*
* if
* base address of high range is below 4Gb
* (bits [47:27] at [31:11])
* DRAM address space on this DCT is hoisted above 4Gb &&
* sys_addr > 4Gb
*
* remove hole offset from sys_addr
* else
* remove high range offset from sys_addr
*/
if ((!(dct_sel_base_addr >> 16) ||
dct_sel_base_addr < dhar_base(pvt)) &&
dhar_valid(pvt) &&
(sys_addr >= BIT_64(32)))
chan_off = hole_off;
else
chan_off = dct_sel_base_off;
} else {
/*
* if
* we have a valid hole &&
* sys_addr > 4Gb
*
* remove hole
* else
* remove dram base to normalize to DCT address
*/
if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
chan_off = hole_off;
else
chan_off = dram_base;
}
return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
}
/*
* checks if the csrow passed in is marked as SPARED, if so returns the new
* spare row
*/
static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
{
int tmp_cs;
if (online_spare_swap_done(pvt, dct) &&
csrow == online_spare_bad_dramcs(pvt, dct)) {
for_each_chip_select(tmp_cs, dct, pvt) {
if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
csrow = tmp_cs;
break;
}
}
}
return csrow;
}
/*
* Iterate over the DRAM DCT "base" and "mask" registers looking for a
* SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
*
* Return:
* -EINVAL: NOT FOUND
* 0..csrow = Chip-Select Row
*/
static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
{
struct mem_ctl_info *mci;
struct amd64_pvt *pvt;
u64 cs_base, cs_mask;
int cs_found = -EINVAL;
int csrow;
mci = edac_mc_find(nid);
if (!mci)
return cs_found;
pvt = mci->pvt_info;
edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
for_each_chip_select(csrow, dct, pvt) {
if (!csrow_enabled(csrow, dct, pvt))
continue;
get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
csrow, cs_base, cs_mask);
cs_mask = ~cs_mask;
edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
(in_addr & cs_mask), (cs_base & cs_mask));
if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
if (pvt->fam == 0x15 && pvt->model >= 0x30) {
cs_found = csrow;
break;
}
cs_found = f10_process_possible_spare(pvt, dct, csrow);
edac_dbg(1, " MATCH csrow=%d\n", cs_found);
break;
}
}
return cs_found;
}
/*
* See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
* swapped with a region located at the bottom of memory so that the GPU can use
* the interleaved region and thus two channels.
*/
static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
{
u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
if (pvt->fam == 0x10) {
/* only revC3 and revE have that feature */
if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
return sys_addr;
}
amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
if (!(swap_reg & 0x1))
return sys_addr;
swap_base = (swap_reg >> 3) & 0x7f;
swap_limit = (swap_reg >> 11) & 0x7f;
rgn_size = (swap_reg >> 20) & 0x7f;
tmp_addr = sys_addr >> 27;
if (!(sys_addr >> 34) &&
(((tmp_addr >= swap_base) &&
(tmp_addr <= swap_limit)) ||
(tmp_addr < rgn_size)))
return sys_addr ^ (u64)swap_base << 27;
return sys_addr;
}
/* For a given @dram_range, check if @sys_addr falls within it. */
static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
u64 sys_addr, int *chan_sel)
{
int cs_found = -EINVAL;
u64 chan_addr;
u32 dct_sel_base;
u8 channel;
bool high_range = false;
u8 node_id = dram_dst_node(pvt, range);
u8 intlv_en = dram_intlv_en(pvt, range);
u32 intlv_sel = dram_intlv_sel(pvt, range);
edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
range, sys_addr, get_dram_limit(pvt, range));
if (dhar_valid(pvt) &&
dhar_base(pvt) <= sys_addr &&
sys_addr < BIT_64(32)) {
amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
sys_addr);
return -EINVAL;
}
if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
return -EINVAL;
sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
dct_sel_base = dct_sel_baseaddr(pvt);
/*
* check whether addresses >= DctSelBaseAddr[47:27] are to be used to
* select between DCT0 and DCT1.
*/
if (dct_high_range_enabled(pvt) &&
!dct_ganging_enabled(pvt) &&
((sys_addr >> 27) >= (dct_sel_base >> 11)))
high_range = true;
channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
high_range, dct_sel_base);
/* Remove node interleaving, see F1x120 */
if (intlv_en)
chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
(chan_addr & 0xfff);
/* remove channel interleave */
if (dct_interleave_enabled(pvt) &&
!dct_high_range_enabled(pvt) &&
!dct_ganging_enabled(pvt)) {
if (dct_sel_interleave_addr(pvt) != 1) {
if (dct_sel_interleave_addr(pvt) == 0x3)
/* hash 9 */
chan_addr = ((chan_addr >> 10) << 9) |
(chan_addr & 0x1ff);
else
/* A[6] or hash 6 */
chan_addr = ((chan_addr >> 7) << 6) |
(chan_addr & 0x3f);
} else
/* A[12] */
chan_addr = ((chan_addr >> 13) << 12) |
(chan_addr & 0xfff);
}
edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
if (cs_found >= 0)
*chan_sel = channel;
return cs_found;
}
static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
u64 sys_addr, int *chan_sel)
{
int cs_found = -EINVAL;
int num_dcts_intlv = 0;
u64 chan_addr, chan_offset;
u64 dct_base, dct_limit;
u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
u64 dhar_offset = f10_dhar_offset(pvt);
u8 intlv_addr = dct_sel_interleave_addr(pvt);
u8 node_id = dram_dst_node(pvt, range);
u8 intlv_en = dram_intlv_en(pvt, range);
amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
range, sys_addr, get_dram_limit(pvt, range));
if (!(get_dram_base(pvt, range) <= sys_addr) &&
!(get_dram_limit(pvt, range) >= sys_addr))
return -EINVAL;
if (dhar_valid(pvt) &&
dhar_base(pvt) <= sys_addr &&
sys_addr < BIT_64(32)) {
amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
sys_addr);
return -EINVAL;
}
/* Verify sys_addr is within DCT Range. */
dct_base = (u64) dct_sel_baseaddr(pvt);
dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
if (!(dct_cont_base_reg & BIT(0)) &&
!(dct_base <= (sys_addr >> 27) &&
dct_limit >= (sys_addr >> 27)))
return -EINVAL;
/* Verify number of dct's that participate in channel interleaving. */
num_dcts_intlv = (int) hweight8(intlv_en);
if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
return -EINVAL;
if (pvt->model >= 0x60)
channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
else
channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
num_dcts_intlv, dct_sel);
/* Verify we stay within the MAX number of channels allowed */
if (channel > 3)
return -EINVAL;
leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
/* Get normalized DCT addr */
if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
chan_offset = dhar_offset;
else
chan_offset = dct_base << 27;
chan_addr = sys_addr - chan_offset;
/* remove channel interleave */
if (num_dcts_intlv == 2) {
if (intlv_addr == 0x4)
chan_addr = ((chan_addr >> 9) << 8) |
(chan_addr & 0xff);
else if (intlv_addr == 0x5)
chan_addr = ((chan_addr >> 10) << 9) |
(chan_addr & 0x1ff);
else
return -EINVAL;
} else if (num_dcts_intlv == 4) {
if (intlv_addr == 0x4)
chan_addr = ((chan_addr >> 10) << 8) |
(chan_addr & 0xff);
else if (intlv_addr == 0x5)
chan_addr = ((chan_addr >> 11) << 9) |
(chan_addr & 0x1ff);
else
return -EINVAL;
}
if (dct_offset_en) {
amd64_read_pci_cfg(pvt->F1,
DRAM_CONT_HIGH_OFF + (int) channel * 4,
&tmp);
chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
}
f15h_select_dct(pvt, channel);
edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
/*
* Find Chip select:
* if channel = 3, then alias it to 1. This is because, in F15 M30h,
* there is support for 4 DCT's, but only 2 are currently functional.
* They are DCT0 and DCT3. But we have read all registers of DCT3 into
* pvt->csels[1]. So we need to use '1' here to get correct info.
* Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
*/
alias_channel = (channel == 3) ? 1 : channel;
cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
if (cs_found >= 0)
*chan_sel = alias_channel;
return cs_found;
}
static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
u64 sys_addr,
int *chan_sel)
{
int cs_found = -EINVAL;
unsigned range;
for (range = 0; range < DRAM_RANGES; range++) {
if (!dram_rw(pvt, range))
continue;
if (pvt->fam == 0x15 && pvt->model >= 0x30)
cs_found = f15_m30h_match_to_this_node(pvt, range,
sys_addr,
chan_sel);
else if ((get_dram_base(pvt, range) <= sys_addr) &&
(get_dram_limit(pvt, range) >= sys_addr)) {
cs_found = f1x_match_to_this_node(pvt, range,
sys_addr, chan_sel);
if (cs_found >= 0)
break;
}
}
return cs_found;
}
/*
* For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
* a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
*
* The @sys_addr is usually an error address received from the hardware
* (MCX_ADDR).
*/
static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
struct err_info *err)
{
struct amd64_pvt *pvt = mci->pvt_info;
error_address_to_page_and_offset(sys_addr, err);
err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
if (err->csrow < 0) {
err->err_code = ERR_CSROW;
return;
}
/*
* We need the syndromes for channel detection only when we're
* ganged. Otherwise @chan should already contain the channel at
* this point.
*/
if (dct_ganging_enabled(pvt))
err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
}
/*
* debug routine to display the memory sizes of all logical DIMMs and its
* CSROWs
*/
static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
{
int dimm, size0, size1;
u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
if (pvt->fam == 0xf) {
/* K8 families < revF not supported yet */
if (pvt->ext_model < K8_REV_F)
return;
else
WARN_ON(ctrl != 0);
}
if (pvt->fam == 0x10) {
dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
: pvt->dbam0;
dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
pvt->csels[1].csbases :
pvt->csels[0].csbases;
} else if (ctrl) {
dbam = pvt->dbam0;
dcsb = pvt->csels[1].csbases;
}
edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
ctrl, dbam);
edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
/* Dump memory sizes for DIMM and its CSROWs */
for (dimm = 0; dimm < 4; dimm++) {
size0 = 0;
if (dcsb[dimm*2] & DCSB_CS_ENABLE)
/*
* For F15m60h, we need multiplier for LRDIMM cs_size
* calculation. We pass dimm value to the dbam_to_cs
* mapper so we can find the multiplier from the
* corresponding DCSM.
*/
size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
DBAM_DIMM(dimm, dbam),
dimm);
size1 = 0;
if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
DBAM_DIMM(dimm, dbam),
dimm);
amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
dimm * 2, size0,
dimm * 2 + 1, size1);
}
}
static struct amd64_family_type family_types[] = {
[K8_CPUS] = {
.ctl_name = "K8",
.f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
.f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
.ops = {
.early_channel_count = k8_early_channel_count,
.map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
.dbam_to_cs = k8_dbam_to_chip_select,
}
},
[F10_CPUS] = {
.ctl_name = "F10h",
.f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
.f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
.ops = {
.early_channel_count = f1x_early_channel_count,
.map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
.dbam_to_cs = f10_dbam_to_chip_select,
}
},
[F15_CPUS] = {
.ctl_name = "F15h",
.f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
.f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
.ops = {
.early_channel_count = f1x_early_channel_count,
.map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
.dbam_to_cs = f15_dbam_to_chip_select,
}
},
[F15_M30H_CPUS] = {
.ctl_name = "F15h_M30h",
.f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
.f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
.ops = {
.early_channel_count = f1x_early_channel_count,
.map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
.dbam_to_cs = f16_dbam_to_chip_select,
}
},
[F15_M60H_CPUS] = {
.ctl_name = "F15h_M60h",
.f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
.f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
.ops = {
.early_channel_count = f1x_early_channel_count,
.map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
.dbam_to_cs = f15_m60h_dbam_to_chip_select,
}
},
[F16_CPUS] = {
.ctl_name = "F16h",
.f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
.f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
.ops = {
.early_channel_count = f1x_early_channel_count,
.map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
.dbam_to_cs = f16_dbam_to_chip_select,
}
},
[F16_M30H_CPUS] = {
.ctl_name = "F16h_M30h",
.f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
.f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
.ops = {
.early_channel_count = f1x_early_channel_count,
.map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
.dbam_to_cs = f16_dbam_to_chip_select,
}
},
[F17_CPUS] = {
.ctl_name = "F17h",
.f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
.f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
.ops = {
.early_channel_count = f17_early_channel_count,
.dbam_to_cs = f17_base_addr_to_cs_size,
}
},
};
/*
* These are tables of eigenvectors (one per line) which can be used for the
* construction of the syndrome tables. The modified syndrome search algorithm
* uses those to find the symbol in error and thus the DIMM.
*
* Algorithm courtesy of Ross LaFetra from AMD.
*/
static const u16 x4_vectors[] = {
0x2f57, 0x1afe, 0x66cc, 0xdd88,
0x11eb, 0x3396, 0x7f4c, 0xeac8,
0x0001, 0x0002, 0x0004, 0x0008,
0x1013, 0x3032, 0x4044, 0x8088,
0x106b, 0x30d6, 0x70fc, 0xe0a8,
0x4857, 0xc4fe, 0x13cc, 0x3288,
0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
0x1f39, 0x251e, 0xbd6c, 0x6bd8,
0x15c1, 0x2a42, 0x89ac, 0x4758,
0x2b03, 0x1602, 0x4f0c, 0xca08,
0x1f07, 0x3a0e, 0x6b04, 0xbd08,
0x8ba7, 0x465e, 0x244c, 0x1cc8,
0x2b87, 0x164e, 0x642c, 0xdc18,
0x40b9, 0x80de, 0x1094, 0x20e8,
0x27db, 0x1eb6, 0x9dac, 0x7b58,
0x11c1, 0x2242, 0x84ac, 0x4c58,
0x1be5, 0x2d7a, 0x5e34, 0xa718,
0x4b39, 0x8d1e, 0x14b4, 0x28d8,
0x4c97, 0xc87e, 0x11fc, 0x33a8,
0x8e97, 0x497e, 0x2ffc, 0x1aa8,
0x16b3, 0x3d62, 0x4f34, 0x8518,
0x1e2f, 0x391a, 0x5cac, 0xf858,
0x1d9f, 0x3b7a, 0x572c, 0xfe18,
0x15f5, 0x2a5a, 0x5264, 0xa3b8,
0x1dbb, 0x3b66, 0x715c, 0xe3f8,
0x4397, 0xc27e, 0x17fc, 0x3ea8,
0x1617, 0x3d3e, 0x6464, 0xb8b8,
0x23ff, 0x12aa, 0xab6c, 0x56d8,
0x2dfb, 0x1ba6, 0x913c, 0x7328,
0x185d, 0x2ca6, 0x7914, 0x9e28,
0x171b, 0x3e36, 0x7d7c, 0xebe8,
0x4199, 0x82ee, 0x19f4, 0x2e58,
0x4807, 0xc40e, 0x130c, 0x3208,
0x1905, 0x2e0a, 0x5804, 0xac08,
0x213f, 0x132a, 0xadfc, 0x5ba8,
0x19a9, 0x2efe, 0xb5cc, 0x6f88,
};
static const u16 x8_vectors[] = {
0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
};
static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
unsigned v_dim)
{
unsigned int i, err_sym;
for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
u16 s = syndrome;
unsigned v_idx = err_sym * v_dim;
unsigned v_end = (err_sym + 1) * v_dim;
/* walk over all 16 bits of the syndrome */
for (i = 1; i < (1U << 16); i <<= 1) {
/* if bit is set in that eigenvector... */
if (v_idx < v_end && vectors[v_idx] & i) {
u16 ev_comp = vectors[v_idx++];
/* ... and bit set in the modified syndrome, */
if (s & i) {
/* remove it. */
s ^= ev_comp;
if (!s)
return err_sym;
}
} else if (s & i)
/* can't get to zero, move to next symbol */
break;
}
}
edac_dbg(0, "syndrome(%x) not found\n", syndrome);
return -1;
}
static int map_err_sym_to_channel(int err_sym, int sym_size)
{
if (sym_size == 4)
switch (err_sym) {
case 0x20:
case 0x21:
return 0;
break;
case 0x22:
case 0x23:
return 1;
break;
default:
return err_sym >> 4;
break;
}
/* x8 symbols */
else
switch (err_sym) {
/* imaginary bits not in a DIMM */
case 0x10:
WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
err_sym);
return -1;
break;
case 0x11:
return 0;
break;
case 0x12:
return 1;
break;
default:
return err_sym >> 3;
break;
}
return -1;
}
static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
{
struct amd64_pvt *pvt = mci->pvt_info;
int err_sym = -1;
if (pvt->ecc_sym_sz == 8)
err_sym = decode_syndrome(syndrome, x8_vectors,
ARRAY_SIZE(x8_vectors),
pvt->ecc_sym_sz);
else if (pvt->ecc_sym_sz == 4)
err_sym = decode_syndrome(syndrome, x4_vectors,
ARRAY_SIZE(x4_vectors),
pvt->ecc_sym_sz);
else {
amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
return err_sym;
}
return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
}
static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
u8 ecc_type)
{
enum hw_event_mc_err_type err_type;
const char *string;
if (ecc_type == 2)
err_type = HW_EVENT_ERR_CORRECTED;
else if (ecc_type == 1)
err_type = HW_EVENT_ERR_UNCORRECTED;
else if (ecc_type == 3)
err_type = HW_EVENT_ERR_DEFERRED;
else {
WARN(1, "Something is rotten in the state of Denmark.\n");
return;
}
switch (err->err_code) {
case DECODE_OK:
string = "";
break;
case ERR_NODE:
string = "Failed to map error addr to a node";
break;
case ERR_CSROW:
string = "Failed to map error addr to a csrow";
break;
case ERR_CHANNEL:
string = "unknown syndrome - possible error reporting race";
break;
default:
string = "WTF error";
break;
}
edac_mc_handle_error(err_type, mci, 1,
err->page, err->offset, err->syndrome,
err->csrow, err->channel, -1,
string, "");
}
static inline void decode_bus_error(int node_id, struct mce *m)
{
struct mem_ctl_info *mci;
struct amd64_pvt *pvt;
u8 ecc_type = (m->status >> 45) & 0x3;
u8 xec = XEC(m->status, 0x1f);
u16 ec = EC(m->status);
u64 sys_addr;
struct err_info err;
mci = edac_mc_find(node_id);
if (!mci)
return;
pvt = mci->pvt_info;
/* Bail out early if this was an 'observed' error */
if (PP(ec) == NBSL_PP_OBS)
return;
/* Do only ECC errors */
if (xec && xec != F10_NBSL_EXT_ERR_ECC)
return;
memset(&err, 0, sizeof(err));
sys_addr = get_error_address(pvt, m);
if (ecc_type == 2)
err.syndrome = extract_syndrome(m->status);
pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
__log_ecc_error(mci, &err, ecc_type);
}
/*
* Use pvt->F3 which contains the F3 CPU PCI device to get the related
* F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
* Reserve F0 and F6 on systems with a UMC.
*/
static int
reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
{
if (pvt->umc) {
pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
if (!pvt->F0) {
amd64_err("error F0 device not found: vendor %x device 0x%x (broken BIOS?)\n",
PCI_VENDOR_ID_AMD, pci_id1);
return -ENODEV;
}
pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
if (!pvt->F6) {
pci_dev_put(pvt->F0);
pvt->F0 = NULL;
amd64_err("error F6 device not found: vendor %x device 0x%x (broken BIOS?)\n",
PCI_VENDOR_ID_AMD, pci_id2);
return -ENODEV;
}
edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
return 0;
}
/* Reserve the ADDRESS MAP Device */
pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
if (!pvt->F1) {
amd64_err("error address map device not found: vendor %x device 0x%x (broken BIOS?)\n",
PCI_VENDOR_ID_AMD, pci_id1);
return -ENODEV;
}
/* Reserve the DCT Device */
pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
if (!pvt->F2) {
pci_dev_put(pvt->F1);
pvt->F1 = NULL;
amd64_err("error F2 device not found: vendor %x device 0x%x (broken BIOS?)\n",
PCI_VENDOR_ID_AMD, pci_id2);
return -ENODEV;
}
edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
return 0;
}
static void free_mc_sibling_devs(struct amd64_pvt *pvt)
{
if (pvt->umc) {
pci_dev_put(pvt->F0);
pci_dev_put(pvt->F6);
} else {
pci_dev_put(pvt->F1);
pci_dev_put(pvt->F2);
}
}
static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
{
pvt->ecc_sym_sz = 4;
if (pvt->umc) {
u8 i;
for (i = 0; i < NUM_UMCS; i++) {
/* Check enabled channels only: */
if ((pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) &&
(pvt->umc[i].ecc_ctrl & BIT(7))) {
pvt->ecc_sym_sz = 8;
break;
}
}
return;
}
if (pvt->fam >= 0x10) {
u32 tmp;
amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
/* F16h has only DCT0, so no need to read dbam1. */
if (pvt->fam != 0x16)
amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
/* F10h, revD and later can do x8 ECC too. */
if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
pvt->ecc_sym_sz = 8;
}
}
/*
* Retrieve the hardware registers of the memory controller.
*/
static void __read_mc_regs_df(struct amd64_pvt *pvt)
{
u8 nid = pvt->mc_node_id;
struct amd64_umc *umc;
u32 i, umc_base;
/* Read registers from each UMC */
for (i = 0; i < NUM_UMCS; i++) {
umc_base = get_umc_base(i);
umc = &pvt->umc[i];
amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
}
}
/*
* Retrieve the hardware registers of the memory controller (this includes the
* 'Address Map' and 'Misc' device regs)
*/
static void read_mc_regs(struct amd64_pvt *pvt)
{
unsigned int range;
u64 msr_val;
/*
* Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
* those are Read-As-Zero.
*/
rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
/* Check first whether TOP_MEM2 is enabled: */
rdmsrl(MSR_K8_SYSCFG, msr_val);
if (msr_val & BIT(21)) {
rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
} else {
edac_dbg(0, " TOP_MEM2 disabled\n");
}
if (pvt->umc) {
__read_mc_regs_df(pvt);
amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
goto skip;
}
amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
read_dram_ctl_register(pvt);
for (range = 0; range < DRAM_RANGES; range++) {
u8 rw;
/* read settings for this DRAM range */
read_dram_base_limit_regs(pvt, range);
rw = dram_rw(pvt, range);
if (!rw)
continue;
edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
range,
get_dram_base(pvt, range),
get_dram_limit(pvt, range));
edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
(rw & 0x1) ? "R" : "-",
(rw & 0x2) ? "W" : "-",
dram_intlv_sel(pvt, range),
dram_dst_node(pvt, range));
}
amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
if (!dct_ganging_enabled(pvt)) {
amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
}
skip:
read_dct_base_mask(pvt);
determine_memory_type(pvt);
edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
determine_ecc_sym_sz(pvt);
dump_misc_regs(pvt);
}
/*
* NOTE: CPU Revision Dependent code
*
* Input:
* @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
* k8 private pointer to -->
* DRAM Bank Address mapping register
* node_id
* DCL register where dual_channel_active is
*
* The DBAM register consists of 4 sets of 4 bits each definitions:
*
* Bits: CSROWs
* 0-3 CSROWs 0 and 1
* 4-7 CSROWs 2 and 3
* 8-11 CSROWs 4 and 5
* 12-15 CSROWs 6 and 7
*
* Values range from: 0 to 15
* The meaning of the values depends on CPU revision and dual-channel state,
* see relevant BKDG more info.
*
* The memory controller provides for total of only 8 CSROWs in its current
* architecture. Each "pair" of CSROWs normally represents just one DIMM in
* single channel or two (2) DIMMs in dual channel mode.
*
* The following code logic collapses the various tables for CSROW based on CPU
* revision.
*
* Returns:
* The number of PAGE_SIZE pages on the specified CSROW number it
* encompasses
*
*/
static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
{
u32 cs_mode, nr_pages;
u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
/*
* The math on this doesn't look right on the surface because x/2*4 can
* be simplified to x*2 but this expression makes use of the fact that
* it is integral math where 1/2=0. This intermediate value becomes the
* number of bits to shift the DBAM register to extract the proper CSROW
* field.
*/
cs_mode = DBAM_DIMM(csrow_nr / 2, dbam);
nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, (csrow_nr / 2))
<< (20 - PAGE_SHIFT);
edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
csrow_nr, dct, cs_mode);
edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
return nr_pages;
}
/*
* Initialize the array of csrow attribute instances, based on the values
* from pci config hardware registers.
*/
static int init_csrows(struct mem_ctl_info *mci)
{
struct amd64_pvt *pvt = mci->pvt_info;
enum edac_type edac_mode = EDAC_NONE;
struct csrow_info *csrow;
struct dimm_info *dimm;
int i, j, empty = 1;
int nr_pages = 0;
u32 val;
if (!pvt->umc) {
amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
pvt->nbcfg = val;
edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
pvt->mc_node_id, val,
!!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
}
/*
* We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
*/
for_each_chip_select(i, 0, pvt) {
bool row_dct0 = !!csrow_enabled(i, 0, pvt);
bool row_dct1 = false;
if (pvt->fam != 0xf)
row_dct1 = !!csrow_enabled(i, 1, pvt);
if (!row_dct0 && !row_dct1)
continue;
csrow = mci->csrows[i];
empty = 0;
edac_dbg(1, "MC node: %d, csrow: %d\n",
pvt->mc_node_id, i);
if (row_dct0) {
nr_pages = get_csrow_nr_pages(pvt, 0, i);
csrow->channels[0]->dimm->nr_pages = nr_pages;
}
/* K8 has only one DCT */
if (pvt->fam != 0xf && row_dct1) {
int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
nr_pages += row_dct1_pages;
}
edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
/* Determine DIMM ECC mode: */
if (pvt->umc) {
if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED)
edac_mode = EDAC_S4ECD4ED;
else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED)
edac_mode = EDAC_SECDED;
} else if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
? EDAC_S4ECD4ED
: EDAC_SECDED;
}
for (j = 0; j < pvt->channel_count; j++) {
dimm = csrow->channels[j]->dimm;
dimm->mtype = pvt->dram_type;
dimm->edac_mode = edac_mode;
}
}
return empty;
}
/* get all cores on this DCT */
static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
{
int cpu;
for_each_online_cpu(cpu)
if (amd_get_nb_id(cpu) == nid)
cpumask_set_cpu(cpu, mask);
}
/* check MCG_CTL on all the cpus on this node */
static bool nb_mce_bank_enabled_on_node(u16 nid)
{
cpumask_var_t mask;
int cpu, nbe;
bool ret = false;
if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
amd64_warn("%s: Error allocating mask\n", __func__);
return false;
}
get_cpus_on_this_dct_cpumask(mask, nid);
rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
for_each_cpu(cpu, mask) {
struct msr *reg = per_cpu_ptr(msrs, cpu);
nbe = reg->l & MSR_MCGCTL_NBE;
edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
cpu, reg->q,
(nbe ? "enabled" : "disabled"));
if (!nbe)
goto out;
}
ret = true;
out:
free_cpumask_var(mask);
return ret;
}
static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
{
cpumask_var_t cmask;
int cpu;
if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
amd64_warn("%s: error allocating mask\n", __func__);
return false;
}
get_cpus_on_this_dct_cpumask(cmask, nid);
rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
for_each_cpu(cpu, cmask) {
struct msr *reg = per_cpu_ptr(msrs, cpu);
if (on) {
if (reg->l & MSR_MCGCTL_NBE)
s->flags.nb_mce_enable = 1;
reg->l |= MSR_MCGCTL_NBE;
} else {
/*
* Turn off NB MCE reporting only when it was off before
*/
if (!s->flags.nb_mce_enable)
reg->l &= ~MSR_MCGCTL_NBE;
}
}
wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
free_cpumask_var(cmask);
return 0;
}
static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
struct pci_dev *F3)
{
bool ret = true;
u32 value, mask = 0x3; /* UECC/CECC enable */
if (toggle_ecc_err_reporting(s, nid, ON)) {
amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
return false;
}
amd64_read_pci_cfg(F3, NBCTL, &value);
s->old_nbctl = value & mask;
s->nbctl_valid = true;
value |= mask;
amd64_write_pci_cfg(F3, NBCTL, value);
amd64_read_pci_cfg(F3, NBCFG, &value);
edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
nid, value, !!(value & NBCFG_ECC_ENABLE));
if (!(value & NBCFG_ECC_ENABLE)) {
amd64_warn("DRAM ECC disabled on this node, enabling...\n");
s->flags.nb_ecc_prev = 0;
/* Attempt to turn on DRAM ECC Enable */
value |= NBCFG_ECC_ENABLE;
amd64_write_pci_cfg(F3, NBCFG, value);
amd64_read_pci_cfg(F3, NBCFG, &value);
if (!(value & NBCFG_ECC_ENABLE)) {
amd64_warn("Hardware rejected DRAM ECC enable,"
"check memory DIMM configuration.\n");
ret = false;
} else {
amd64_info("Hardware accepted DRAM ECC Enable\n");
}
} else {
s->flags.nb_ecc_prev = 1;
}
edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
nid, value, !!(value & NBCFG_ECC_ENABLE));
return ret;
}
static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
struct pci_dev *F3)
{
u32 value, mask = 0x3; /* UECC/CECC enable */
if (!s->nbctl_valid)
return;
amd64_read_pci_cfg(F3, NBCTL, &value);
value &= ~mask;
value |= s->old_nbctl;
amd64_write_pci_cfg(F3, NBCTL, value);
/* restore previous BIOS DRAM ECC "off" setting we force-enabled */
if (!s->flags.nb_ecc_prev) {
amd64_read_pci_cfg(F3, NBCFG, &value);
value &= ~NBCFG_ECC_ENABLE;
amd64_write_pci_cfg(F3, NBCFG, value);
}
/* restore the NB Enable MCGCTL bit */
if (toggle_ecc_err_reporting(s, nid, OFF))
amd64_warn("Error restoring NB MCGCTL settings!\n");
}
/*
* EDAC requires that the BIOS have ECC enabled before
* taking over the processing of ECC errors. A command line
* option allows to force-enable hardware ECC later in
* enable_ecc_error_reporting().
*/
static const char *ecc_msg =
"ECC disabled in the BIOS or no ECC capability, module will not load.\n"
" Either enable ECC checking or force module loading by setting "
"'ecc_enable_override'.\n"
" (Note that use of the override may cause unknown side effects.)\n";
static bool ecc_enabled(struct pci_dev *F3, u16 nid)
{
bool nb_mce_en = false;
u8 ecc_en = 0, i;
u32 value;
if (boot_cpu_data.x86 >= 0x17) {
u8 umc_en_mask = 0, ecc_en_mask = 0;
for (i = 0; i < NUM_UMCS; i++) {
u32 base = get_umc_base(i);
/* Only check enabled UMCs. */
if (amd_smn_read(nid, base + UMCCH_SDP_CTRL, &value))
continue;
if (!(value & UMC_SDP_INIT))
continue;
umc_en_mask |= BIT(i);
if (amd_smn_read(nid, base + UMCCH_UMC_CAP_HI, &value))
continue;
if (value & UMC_ECC_ENABLED)
ecc_en_mask |= BIT(i);
}
/* Check whether at least one UMC is enabled: */
if (umc_en_mask)
ecc_en = umc_en_mask == ecc_en_mask;
/* Assume UMC MCA banks are enabled. */
nb_mce_en = true;
} else {
amd64_read_pci_cfg(F3, NBCFG, &value);
ecc_en = !!(value & NBCFG_ECC_ENABLE);
nb_mce_en = nb_mce_bank_enabled_on_node(nid);
if (!nb_mce_en)
amd64_notice("NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
MSR_IA32_MCG_CTL, nid);
}
amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
if (!ecc_en || !nb_mce_en) {
amd64_notice("%s", ecc_msg);
return false;
}
return true;
}
static inline void
f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
{
u8 i, ecc_en = 1, cpk_en = 1;
for (i = 0; i < NUM_UMCS; i++) {
if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
}
}
/* Set chipkill only if ECC is enabled: */
if (ecc_en) {
mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
if (cpk_en)
mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
}
}
static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
struct amd64_family_type *fam)
{
struct amd64_pvt *pvt = mci->pvt_info;
mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
mci->edac_ctl_cap = EDAC_FLAG_NONE;
if (pvt->umc) {
f17h_determine_edac_ctl_cap(mci, pvt);
} else {
if (pvt->nbcap & NBCAP_SECDED)
mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
if (pvt->nbcap & NBCAP_CHIPKILL)
mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
}
mci->edac_cap = determine_edac_cap(pvt);
mci->mod_name = EDAC_MOD_STR;
mci->mod_ver = EDAC_AMD64_VERSION;
mci->ctl_name = fam->ctl_name;
mci->dev_name = pci_name(pvt->F3);
mci->ctl_page_to_phys = NULL;
/* memory scrubber interface */
mci->set_sdram_scrub_rate = set_scrub_rate;
mci->get_sdram_scrub_rate = get_scrub_rate;
}
/*
* returns a pointer to the family descriptor on success, NULL otherwise.
*/
static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
{
struct amd64_family_type *fam_type = NULL;
pvt->ext_model = boot_cpu_data.x86_model >> 4;
pvt->stepping = boot_cpu_data.x86_mask;
pvt->model = boot_cpu_data.x86_model;
pvt->fam = boot_cpu_data.x86;
switch (pvt->fam) {
case 0xf:
fam_type = &family_types[K8_CPUS];
pvt->ops = &family_types[K8_CPUS].ops;
break;
case 0x10:
fam_type = &family_types[F10_CPUS];
pvt->ops = &family_types[F10_CPUS].ops;
break;
case 0x15:
if (pvt->model == 0x30) {
fam_type = &family_types[F15_M30H_CPUS];
pvt->ops = &family_types[F15_M30H_CPUS].ops;
break;
} else if (pvt->model == 0x60) {
fam_type = &family_types[F15_M60H_CPUS];
pvt->ops = &family_types[F15_M60H_CPUS].ops;
break;
}
fam_type = &family_types[F15_CPUS];
pvt->ops = &family_types[F15_CPUS].ops;
break;
case 0x16:
if (pvt->model == 0x30) {
fam_type = &family_types[F16_M30H_CPUS];
pvt->ops = &family_types[F16_M30H_CPUS].ops;
break;
}
fam_type = &family_types[F16_CPUS];
pvt->ops = &family_types[F16_CPUS].ops;
break;
case 0x17:
fam_type = &family_types[F17_CPUS];
pvt->ops = &family_types[F17_CPUS].ops;
break;
default:
amd64_err("Unsupported family!\n");
return NULL;
}
amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
(pvt->fam == 0xf ?
(pvt->ext_model >= K8_REV_F ? "revF or later "
: "revE or earlier ")
: ""), pvt->mc_node_id);
return fam_type;
}
static const struct attribute_group *amd64_edac_attr_groups[] = {
#ifdef CONFIG_EDAC_DEBUG
&amd64_edac_dbg_group,
#endif
#ifdef CONFIG_EDAC_AMD64_ERROR_INJECTION
&amd64_edac_inj_group,
#endif
NULL
};
static int init_one_instance(unsigned int nid)
{
struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
struct amd64_family_type *fam_type = NULL;
struct mem_ctl_info *mci = NULL;
struct edac_mc_layer layers[2];
struct amd64_pvt *pvt = NULL;
u16 pci_id1, pci_id2;
int err = 0, ret;
ret = -ENOMEM;
pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
if (!pvt)
goto err_ret;
pvt->mc_node_id = nid;
pvt->F3 = F3;
ret = -EINVAL;
fam_type = per_family_init(pvt);
if (!fam_type)
goto err_free;
if (pvt->fam >= 0x17) {
pvt->umc = kcalloc(NUM_UMCS, sizeof(struct amd64_umc), GFP_KERNEL);
if (!pvt->umc) {
ret = -ENOMEM;
goto err_free;
}
pci_id1 = fam_type->f0_id;
pci_id2 = fam_type->f6_id;
} else {
pci_id1 = fam_type->f1_id;
pci_id2 = fam_type->f2_id;
}
err = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
if (err)
goto err_post_init;
read_mc_regs(pvt);
/*
* We need to determine how many memory channels there are. Then use
* that information for calculating the size of the dynamic instance
* tables in the 'mci' structure.
*/
ret = -EINVAL;
pvt->channel_count = pvt->ops->early_channel_count(pvt);
if (pvt->channel_count < 0)
goto err_siblings;
ret = -ENOMEM;
layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
layers[0].size = pvt->csels[0].b_cnt;
layers[0].is_virt_csrow = true;
layers[1].type = EDAC_MC_LAYER_CHANNEL;
/*
* Always allocate two channels since we can have setups with DIMMs on
* only one channel. Also, this simplifies handling later for the price
* of a couple of KBs tops.
*/
layers[1].size = 2;
layers[1].is_virt_csrow = false;
mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
if (!mci)
goto err_siblings;
mci->pvt_info = pvt;
mci->pdev = &pvt->F3->dev;
setup_mci_misc_attrs(mci, fam_type);
if (init_csrows(mci))
mci->edac_cap = EDAC_FLAG_NONE;
ret = -ENODEV;
if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
edac_dbg(1, "failed edac_mc_add_mc()\n");
goto err_add_mc;
}
/* register stuff with EDAC MCE */
if (report_gart_errors)
amd_report_gart_errors(true);
amd_register_ecc_decoder(decode_bus_error);
return 0;
err_add_mc:
edac_mc_free(mci);
err_siblings:
free_mc_sibling_devs(pvt);
err_post_init:
if (pvt->fam >= 0x17)
kfree(pvt->umc);
err_free:
kfree(pvt);
err_ret:
return ret;
}
static int probe_one_instance(unsigned int nid)
{
struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
struct ecc_settings *s;
int ret;
ret = -ENOMEM;
s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
if (!s)
goto err_out;
ecc_stngs[nid] = s;
if (!ecc_enabled(F3, nid)) {
ret = -ENODEV;
if (!ecc_enable_override)
goto err_enable;
if (boot_cpu_data.x86 >= 0x17) {
amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
goto err_enable;
} else
amd64_warn("Forcing ECC on!\n");
if (!enable_ecc_error_reporting(s, nid, F3))
goto err_enable;
}
ret = init_one_instance(nid);
if (ret < 0) {
amd64_err("Error probing instance: %d\n", nid);
if (boot_cpu_data.x86 < 0x17)
restore_ecc_error_reporting(s, nid, F3);
}
return ret;
err_enable:
kfree(s);
ecc_stngs[nid] = NULL;
err_out:
return ret;
}
static void remove_one_instance(unsigned int nid)
{
struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
struct ecc_settings *s = ecc_stngs[nid];
struct mem_ctl_info *mci;
struct amd64_pvt *pvt;
mci = find_mci_by_dev(&F3->dev);
WARN_ON(!mci);
/* Remove from EDAC CORE tracking list */
mci = edac_mc_del_mc(&F3->dev);
if (!mci)
return;
pvt = mci->pvt_info;
restore_ecc_error_reporting(s, nid, F3);
free_mc_sibling_devs(pvt);
/* unregister from EDAC MCE */
amd_report_gart_errors(false);
amd_unregister_ecc_decoder(decode_bus_error);
kfree(ecc_stngs[nid]);
ecc_stngs[nid] = NULL;
/* Free the EDAC CORE resources */
mci->pvt_info = NULL;
kfree(pvt);
edac_mc_free(mci);
}
static void setup_pci_device(void)
{
struct mem_ctl_info *mci;
struct amd64_pvt *pvt;
if (pci_ctl)
return;
mci = edac_mc_find(0);
if (!mci)
return;
pvt = mci->pvt_info;
if (pvt->umc)
pci_ctl = edac_pci_create_generic_ctl(&pvt->F0->dev, EDAC_MOD_STR);
else
pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
if (!pci_ctl) {
pr_warn("%s(): Unable to create PCI control\n", __func__);
pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
}
}
static const struct x86_cpu_id amd64_cpuids[] = {
{ X86_VENDOR_AMD, 0xF, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
{ X86_VENDOR_AMD, 0x10, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
{ X86_VENDOR_AMD, 0x15, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
{ X86_VENDOR_AMD, 0x16, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
{ }
};
MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
static int __init amd64_edac_init(void)
{
int err = -ENODEV;
int i;
if (amd_cache_northbridges() < 0)
goto err_ret;
opstate_init();
err = -ENOMEM;
ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
if (!ecc_stngs)
goto err_free;
msrs = msrs_alloc();
if (!msrs)
goto err_free;
for (i = 0; i < amd_nb_num(); i++)
if (probe_one_instance(i)) {
/* unwind properly */
while (--i >= 0)
remove_one_instance(i);
goto err_pci;
}
setup_pci_device();
#ifdef CONFIG_X86_32
amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
#endif
printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
return 0;
err_pci:
msrs_free(msrs);
msrs = NULL;
err_free:
kfree(ecc_stngs);
ecc_stngs = NULL;
err_ret:
return err;
}
static void __exit amd64_edac_exit(void)
{
int i;
if (pci_ctl)
edac_pci_release_generic_ctl(pci_ctl);
for (i = 0; i < amd_nb_num(); i++)
remove_one_instance(i);
kfree(ecc_stngs);
ecc_stngs = NULL;
msrs_free(msrs);
msrs = NULL;
}
module_init(amd64_edac_init);
module_exit(amd64_edac_exit);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
"Dave Peterson, Thayne Harbaugh");
MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
EDAC_AMD64_VERSION);
module_param(edac_op_state, int, 0444);
MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
Reference in New Issue View Git Blame Copy Permalink