src/northbridge/intel/i3100/raminit_ep80579.c - chromiumos/third_party/coreboot - Git at Google

 /*
  * This file is part of the coreboot project.
  *
  * Copyright (C) 2005 Eric W. Biederman and Tom Zimmerman
  * Copyright (C) 2008 Arastra, Inc.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  */

 #include <cpu/x86/mtrr.h>
 #include <cpu/x86/cache.h>
 #include "raminit_ep80579.h"
 #include "ep80579.h"

 #define BAR 0x90000000

 static void sdram_set_registers(const struct mem_controller *ctrl)
 {
 	static const u32 register_values[] = {
 		PCI_ADDR(0, 0x00, 0, CKDIS), 0xffff0000, 0x0000ffff,
 		PCI_ADDR(0, 0x00, 0, DEVPRES), 0x00000000, 0x07420001 | DEVPRES_CONFIG,
 		PCI_ADDR(0, 0x00, 0, PAM-1), 0xcccccc7f, 0x33333000,
 		PCI_ADDR(0, 0x00, 0, PAM+3), 0xcccccccc, 0x33333333,
 		PCI_ADDR(0, 0x00, 0, DEVPRES1), 0xffffffff, 0x0040003a,
 		PCI_ADDR(0, 0x00, 0, SMRBASE), 0x00000fff, BAR | 0,
 	};
 	int i;
 	int max;

 	for (i = 0; i < ARRAY_SIZE(register_values); i += 3) {
 		device_t dev;
 		u32 where;
 		u32 reg;
 		dev = (register_values[i] & ~0xff) - PCI_DEV(0, 0x00, 0) + ctrl->f0;
 		where = register_values[i] & 0xff;
 		reg = pci_read_config32(dev, where);
 		reg &= register_values[i+1];
 		reg |= register_values[i+2];
 		pci_write_config32(dev, where, reg);
 	}
 }

 struct dimm_size {
 	u32 side1;
 	u32 side2;
 };

 static struct dimm_size spd_get_dimm_size(u16 device)
 {
 	/* Calculate the log base 2 size of a DIMM in bits */
 	struct dimm_size sz;
 	int value, low, ddr2;
 	sz.side1 = 0;
 	sz.side2 = 0;

 	/* Note it might be easier to use byte 31 here, it has the DIMM size as
 	 * a multiple of 4MB.  The way we do it now we can size both
 	 * sides of an assymetric dimm.
 	 */
 	value = spd_read_byte(device, SPD_NUM_ROWS);
 	if (value < 0) goto hw_err;
 	if ((value & 0xf) == 0) goto val_err;
 	sz.side1 += value & 0xf;

 	value = spd_read_byte(device, SPD_NUM_COLUMNS);
 	if (value < 0) goto hw_err;
 	if ((value & 0xf) == 0) goto val_err;
 	sz.side1 += value & 0xf;

 	value = spd_read_byte(device, SPD_NUM_BANKS_PER_SDRAM);
 	if (value < 0) goto hw_err;
 	if ((value & 0xff) == 0) goto val_err;
 	sz.side1 += log2(value & 0xff);

 	/* Get the module data width and convert it to a power of two */
 	value = spd_read_byte(device, SPD_MODULE_DATA_WIDTH_MSB);
 	if (value < 0) goto hw_err;
 	value &= 0xff;
 	value <<= 8;

 	low = spd_read_byte(device, SPD_MODULE_DATA_WIDTH_LSB);
 	if (low < 0) goto hw_err;
 	value = value | (low & 0xff);
 	if ((value != 72) && (value != 64)) goto val_err;
 	sz.side1 += log2(value);

 	/* side 2 */
 	value = spd_read_byte(device, SPD_NUM_DIMM_BANKS);

 	if (value < 0) goto hw_err;
 	value &= 7;
 	value++;
 	if (value == 1) goto out;
 	if (value != 2) goto val_err;

 	/* Start with the symmetrical case */
 	sz.side2 = sz.side1;

 	value = spd_read_byte(device, SPD_NUM_ROWS);
 	if (value < 0) goto hw_err;
 	if ((value & 0xf0) == 0) goto out;	/* If symmetrical we are done */
 	sz.side2 -= (value & 0x0f);		/* Subtract out rows on side 1 */
 	sz.side2 += ((value >> 4) & 0x0f);	/* Add in rows on side 2 */

 	value = spd_read_byte(device, SPD_NUM_COLUMNS);
 	if (value < 0) goto hw_err;
 	if ((value & 0xff) == 0) goto val_err;
 	sz.side2 -= (value & 0x0f);		/* Subtract out columns on side 1 */
 	sz.side2 += ((value >> 4) & 0x0f);	/* Add in columns on side 2 */
 	goto out;

  val_err:
 	die("Bad SPD value\n");
 	/* If an hw_error occurs report that I have no memory */
  hw_err:
 	sz.side1 = 0;
 	sz.side2 = 0;
 out:
 	print_debug("dimm ");
 	print_debug_hex8(device);
 	print_debug(" size = ");
 	print_debug_hex8(sz.side1);
 	print_debug(".");
 	print_debug_hex8(sz.side2);
 	print_debug("\n");
 	return sz;

 }

 static long spd_set_ram_size(const struct mem_controller *ctrl, u8 dimm_mask)
 {
 	int i;
 	int cum;

 	for (i = cum = 0; i < DIMM_SOCKETS; i++) {
 		struct dimm_size sz;
 		if (dimm_mask & (1 << i)) {
 			sz = spd_get_dimm_size(ctrl->channel0[i]);
 			if (sz.side1 < 29) {
 				return -1; /* Report SPD error */
 			}
 			/* convert bits to multiples of 64MB */
 			sz.side1 -= 29;
 			cum += (1 << sz.side1);
 			pci_write_config8(ctrl->f0, DRB + (i*2), cum);
 			pci_write_config8(ctrl->f0, DRB+1 + (i*2), cum);
 			if (spd_read_byte(ctrl->channel0[i], SPD_NUM_DIMM_BANKS) & 0x1) {
 				cum <<= 1;
 			}
 		}
 		else {
 			pci_write_config8(ctrl->f0, DRB + (i*2), cum);
 			pci_write_config8(ctrl->f0, DRB+1 + (i*2), cum);
 		}
 	}
 	print_debug("DRB = ");
 	print_debug_hex32(pci_read_config32(ctrl->f0, DRB));
 	print_debug("\n");

 	cum >>= 1;
 	/* set TOM top of memory */
 	pci_write_config16(ctrl->f0, TOM, cum);
 	print_debug("TOM = ");
 	print_debug_hex16(cum);
 	print_debug("\n");
 	/* set TOLM top of low memory */
 	if (cum > 0x18) {
 		cum = 0x18;
 	}
 	cum <<= 11;
 	pci_write_config16(ctrl->f0, TOLM, cum);
 	print_debug("TOLM = ");
 	print_debug_hex16(cum);
 	print_debug("\n");
 	return 0;
 }


 static u8 spd_detect_dimms(const struct mem_controller *ctrl)
 {
 	u8 dimm_mask = 0;
 	int i;
 	for (i = 0; i < DIMM_SOCKETS; i++) {
 		int byte;
 		u16 device;
 		device = ctrl->channel0[i];
 		if (device) {
 			byte = spd_read_byte(device, SPD_MEMORY_TYPE);
 			print_debug("spd ");
 			print_debug_hex8(device);
 			print_debug(" = ");
 			print_debug_hex8(byte);
 			print_debug("\n");
 			if (byte == 8) {
 				dimm_mask |= (1 << i);
 			}
 		}
 	}
 	return dimm_mask;
 }


 static int spd_set_row_attributes(const struct mem_controller *ctrl,
 				  u8 dimm_mask)
 {
 	int value;
 	int i;

 	for (i = 0; i < DIMM_SOCKETS; i++) {
 		u32 dra = 0;
 		int reg = 0;

 		if (!(dimm_mask & (1 << i))) {
 			continue;
 		}

 		value = spd_read_byte(ctrl->channel0[i], SPD_NUM_ROWS);
 		if (value < 0) die("Bad SPD data\n");
 		if ((value & 0xf) == 0) die("Invalid # of rows\n");
 		dra |= (((value-13) & 0x7) << 23);
 		dra |= (((value-13) & 0x7) << 29);
 		reg += value & 0xf;

 		value = spd_read_byte(ctrl->channel0[i], SPD_NUM_COLUMNS);
 		if (value < 0) die("Bad SPD data\n");
 		if ((value & 0xf) == 0) die("Invalid # of columns\n");
 		dra |= (((value-10) & 0x7) << 20);
 		dra |= (((value-10) & 0x7) << 26);
 		reg += value & 0xf;

 		value = spd_read_byte(ctrl->channel0[i], SPD_NUM_BANKS_PER_SDRAM);
 		if (value < 0) die("Bad SPD data\n");
 		if ((value & 0xff) == 0) die("Invalid # of banks\n");
 		reg += log2(value & 0xff);

 		print_debug("dimm ");
 		print_debug_hex8(i);
 		print_debug(" reg = ");
 		print_debug_hex8(reg);
 		print_debug("\n");

 		/* set device density */
 		dra |= ((31-reg));
 		dra |= ((31-reg) << 6);

 		/* set device width (x8) */
 		dra |= (1 << 4);
 		dra |= (1 << 10);

 		/* set device type (registered) */
 		dra |= (1 << 14);

 		/* set number of ranks (0=single, 1=dual) */
 		value = spd_read_byte(ctrl->channel0[i], SPD_NUM_DIMM_BANKS);
 		dra |= ((value & 0x1) << 17);

 		print_debug("DRA");
 		print_debug_hex8(i);
 		print_debug(" = ");
 		print_debug_hex32(dra);
 		print_debug("\n");

 		pci_write_config32(ctrl->f0, DRA + (i*4), dra);
 	}
 	return 0;
 }


 static u32 spd_set_drt_attributes(const struct mem_controller *ctrl,
 		u8 dimm_mask, u32 drc)
 {
 	int i;
 	u32 val, val1;
 	u32 cl;
 	u32 trc = 0;
 	u32 trfc = 0;
 	u32 tras = 0;
 	u32 trtp = 0;
 	u32 twtr = 0;
 	int index = drc & 0x00000003;
 	int ci;
 	static const u8 latencies[] = { /* 533, 800, 400, 667 */
 		0x10, 0x60, 0x10, 0x20 };
 	static const u32 drt0[] = { /* 533, 800, 400, 667 */
 		0x24240002, 0x24360002, 0x24220002, 0x24360002 };
 	static const u32 drt1[] = { /* 533, 800, 400, 667 */
 		0x00400000, 0x00900000, 0x00200000, 0x00700000 };
 	static const u32 magic[] = { /* 533, 800, 400, 667 */
 		0x007b8221, 0x00b94331, 0x005ca1a1, 0x009a62b1 };
 	static const u32 mrs[] = { /* 533, 800, 400, 667 */
 		0x07020000, 0x0b020000, 0x05020000, 0x09020000 };
 	static const int cycle[] = { /* 533, 800, 400, 667 */
 		15, 10, 20, 12 }; /* cycle time in 1/4 ns units */
 	static const int byte40rem[] = {
 		0, 1, 2, 2, 3, 3, 0, 0 }; /* byte 40 remainder in 1/4 ns units */

 	/* CAS latency in cycles */
 	val = latencies[index];
 	for (i = 0; i < DIMM_SOCKETS; i++) {
 		if (!(dimm_mask & (1 << i)))
 			continue;
 		val &= spd_read_byte(ctrl->channel0[i], SPD_ACCEPTABLE_CAS_LATENCIES);
 	}
 	if (val & 0x10)
 		cl = 4;
 	else if (val & 0x20)
 		cl = 5;
 	else if (val & 0x40)
 		cl = 6;
 	else
 		die("CAS latency mismatch\n");
 	print_debug("cl = ");
 	print_debug_hex8(cl);
 	print_debug("\n");

 	ci = cycle[index];

 	/* Trc, Trfc in cycles */
 	for (i = 0; i < DIMM_SOCKETS; i++) {
 		if (!(dimm_mask & (1 << i)))
 			continue;
 		val1 = spd_read_byte(ctrl->channel0[i], SPD_BYTE_41_42_EXTENSION);
 		val = spd_read_byte(ctrl->channel0[i], SPD_MIN_ACT_TO_ACT_AUTO_REFRESH);
 		val <<= 2; /* convert to 1/4 ns */
 		val += byte40rem[(val1 >> 4) & 0x7];
 		val = (val + ci - 1) / ci + 1; /* convert to cycles */
 		if (trc < val)
 			trc = val;
 		val = spd_read_byte(ctrl->channel0[i], SPD_MIN_AUTO_REFRESH_TO_ACT);
 		val <<= 2; /* convert to 1/4 ns */
 		if (val1 & 0x01)
 			val += 1024;
 		val += byte40rem[(val1 >> 1) & 0x7];
 		val = (val + ci - 1) / ci; /* convert to cycles */
 		if (trfc < val)
 			trfc = val;
 	}
 	print_debug("trc = ");
 	print_debug_hex8(trc);
 	print_debug("\n");
 	print_debug("trfc = ");
 	print_debug_hex8(trfc);
 	print_debug("\n");

 	/* Tras, Trtp, Twtr in cycles */
 	for (i = 0; i < DIMM_SOCKETS; i++) {
 		if (!(dimm_mask & (1 << i)))
 			continue;
 		val = spd_read_byte(ctrl->channel0[i], SPD_MIN_ACTIVE_TO_PRECHARGE_DELAY);
 		val <<= 2; /* convert to 1/4 ns */
 		val = (val + ci - 1) / ci; /* convert to cycles */
 		if (tras < val)
 			tras = val;
 		val = spd_read_byte(ctrl->channel0[i], SPD_INT_READ_TO_PRECHARGE_DELAY);
 		val = (val + ci - 1) / ci; /* convert to cycles */
 		if (trtp < val)
 			trtp = val;
 		val = spd_read_byte(ctrl->channel0[i], SPD_INT_WRITE_TO_READ_DELAY);
 		val = (val + ci - 1) / ci; /* convert to cycles */
 		if (twtr < val)
 			twtr = val;
 	}
 	print_debug("tras = ");
 	print_debug_hex8(tras);
 	print_debug("\n");
 	print_debug("trtp = ");
 	print_debug_hex8(trtp);
 	print_debug("\n");
 	print_debug("twtr = ");
 	print_debug_hex8(twtr);
 	print_debug("\n");

 	val = (drt0[index] | ((trc - 11) << 12) | ((cl - 3) << 9)
 	       | ((cl - 3) << 6) | ((cl - 3) << 3));
 	print_debug("drt0 = ");
 	print_debug_hex32(val);
 	print_debug("\n");
 	pci_write_config32(ctrl->f0, DRT0, val);

 	val = (drt1[index] | ((tras - 8) << 28) | ((trtp - 2) << 25)
 	       | (twtr << 15));
 	print_debug("drt1 = ");
 	print_debug_hex32(val);
 	print_debug("\n");
 	pci_write_config32(ctrl->f0, DRT1, val);

 	val = (magic[index]);
 	print_debug("magic = ");
 	print_debug_hex32(val);
 	print_debug("\n");
 	pci_write_config32(PCI_DEV(0, 0x08, 0), 0xcc, val);

 	val = (mrs[index] | (cl << 20));
 	print_debug("mrs = ");
 	print_debug_hex32(val);
 	print_debug("\n");
 	return val;
 }

 static int spd_set_dram_controller_mode(const struct mem_controller *ctrl,
 					u8 dimm_mask)
 {
 	int value;
 	int drc = 0;
 	int i;
 	msr_t msr;
 	u8 cycle = 0x25;

 	for (i = 0; i < DIMM_SOCKETS; i++) {
 		if (!(dimm_mask & (1 << i)))
 			continue;
 		if ((spd_read_byte(ctrl->channel0[i], SPD_MODULE_DATA_WIDTH_LSB) & 0xf0) != 0x40)
 			die("ERROR: Only 64-bit DIMMs supported\n");
 		if (!(spd_read_byte(ctrl->channel0[i], SPD_DIMM_CONFIG_TYPE) & 0x02))
 			die("ERROR: Only ECC DIMMs supported\n");
 		if (spd_read_byte(ctrl->channel0[i], SPD_PRIMARY_SDRAM_WIDTH) != 0x08)
 			die("ERROR: Only x8 DIMMs supported\n");

 		value = spd_read_byte(ctrl->channel0[i], SPD_MIN_CYCLE_TIME_AT_CAS_MAX);
 		if (value > cycle)
 			cycle = value;
 	}
 	print_debug("cycle = ");
 	print_debug_hex8(cycle);
 	print_debug("\n");

 	drc |= (1 << 20); /* enable ECC */
 	drc |= (3 << 30); /* enable CKE on each DIMM */
 	drc |= (1 << 4); /* enable CKE globally */

 	/* TODO check: */
 	/* set front side bus speed */
 	msr = rdmsr(0xcd); /* returns 0 on Pentium M 90nm */
 	print_debug("msr 0xcd = ");
 	print_debug_hex32(msr.hi);
 	print_debug_hex32(msr.lo);
 	print_debug("\n");

 	/* TODO check that this msr really indicates fsb speed! */
 	if (msr.lo & 0x07) {
 		print_info("533 MHz FSB\n");
 		if (cycle <= 0x25) {
 			drc |= 0x5;
 			print_info("400 MHz DDR\n");
 		} else if (cycle <= 0x30) {
 			drc |= 0x7;
 			print_info("333 MHz DDR\n");
 		} else if (cycle <= 0x3d) {
 			drc |= 0x4;
 			print_info("266 MHz DDR\n");
 		} else {
 			drc |= 0x2;
 			print_info("200 MHz DDR\n");
 		}
 	}
 	else {
 		print_info("400 MHz FSB\n");
 		if (cycle <= 0x30) {
 			drc |= 0x7;
 			print_info("333 MHz DDR\n");
 		} else if (cycle <= 0x3d) {
 			drc |= 0x0;
 			print_info("266 MHz DDR\n");
 		} else {
 			drc |= 0x2;
 			print_info("200 MHz DDR\n");
 		}
 	}

 	print_debug("DRC = ");
 	print_debug_hex32(drc);
 	print_debug("\n");

 	return drc;
 }

 static void sdram_set_spd_registers(const struct mem_controller *ctrl)
 {
 	u8 dimm_mask;
 	int i;

 	/* Test if we can read the SPD */
 	dimm_mask = spd_detect_dimms(ctrl);
 	if (!(dimm_mask & ((1 << DIMM_SOCKETS) - 1))) {
 		print_err("No memory for this cpu\n");
 		return;
 	}
 	return;
 }

 static void set_on_dimm_termination_enable(const struct mem_controller *ctrl)
 {
 	u8 c1,c2;
         u32 dimm,i;
         u32 data32;
 	u32 t4;

 	/* Set up northbridge values */
 	/* ODT enable */
   	pci_write_config32(ctrl->f0, SDRC, 0xa0002c30);

 	c1 = pci_read_config8(ctrl->f0, DRB);
 	c2 = pci_read_config8(ctrl->f0, DRB+2);
 	if (c1 == c2) {
 		/* 1 single-rank DIMM */
 		data32 = 0x00000010;
 	}
 	else {
 		/* 2 single-rank DIMMs or 1 double-rank DIMM */
 		data32 = 0x00002010;
 	}

 	print_debug("ODT Value = ");
 	print_debug_hex32(data32);
 	print_debug("\n");

   	pci_write_config32(ctrl->f0, DDR2ODTC, data32);

 	for (i = 0; i < 2; i++) {
 		print_debug("ODT CS");
 		print_debug_hex8(i);
 		print_debug("\n");

 		write32(BAR+DCALADDR, 0x0b840001);
 		write32(BAR+DCALCSR, 0x80000003 | ((i+1)<<21));
 		do data32 = read32(BAR+DCALCSR);
 		while (data32 & 0x80000000);
 	}
 }


 static void dump_dcal_regs(void)
 {
 	int i;
 	for (i = 0x0; i < 0x2a0; i += 4) {
 		if ((i % 16) == 0) {
 			print_debug("\n");
 			print_debug_hex16(i);
 			print_debug(": ");
 		}
 		print_debug_hex32(read32(BAR+i));
 		print_debug(" ");
 	}
 	print_debug("\n");
 }


 static void sdram_enable(int controllers, const struct mem_controller *ctrl)
 {
 	int i;
 	int cs;
 	long mask;
 	u32 drc;
 	u32 data32;
 	u32 mode_reg;
 	msr_t msr;
 	u16 data16;

 	mask = spd_detect_dimms(ctrl);
 	print_debug("Starting SDRAM Enable\n");

 	/* Set DRAM type and Front Side Bus frequency */
 	drc = spd_set_dram_controller_mode(ctrl, mask);
 	if (drc == 0) {
 		die("Error calculating DRC\n");
 	}
 	data32 = drc & ~(3 << 20);  /* clear ECC mode */
 	data32 = data32 | (3 << 5);  /* temp turn off ODT */
   	/* Set DRAM controller mode */
   	pci_write_config32(ctrl->f0, DRC, data32);

 	/* Turn the clocks on */
   	pci_write_config16(ctrl->f0, CKDIS, 0x0000);

 	/* Program row size */
 	spd_set_ram_size(ctrl, mask);

 	/* Program row attributes */
 	spd_set_row_attributes(ctrl, mask);

 	/* Program timing values */
 	mode_reg = spd_set_drt_attributes(ctrl, mask, drc);

 	dump_dcal_regs();

 	/* Apply NOP */
 	for (cs = 0; cs < 2; cs++) {
 		print_debug("NOP CS");
 		print_debug_hex8(cs);
 		print_debug("\n");
 		udelay(16);
 		write32(BAR+DCALCSR, (0x00000000 | ((cs+1)<<21)));
 		write32(BAR+DCALCSR, (0x80000000 | ((cs+1)<<21)));
 		do data32 = read32(BAR+DCALCSR);
 		while (data32 & 0x80000000);
 	}

 	/* Apply NOP */
 	udelay(16);
 	for (cs = 0; cs < 2; cs++) {
 		print_debug("NOP CS");
 		print_debug_hex8(cs);
 		print_debug("\n");
 		write32(BAR + DCALCSR, (0x80000000 | ((cs+1)<<21)));
 		do data32 = read32(BAR+DCALCSR);
 		while (data32 & 0x80000000);
 	}

 	/* Precharge all banks */
 	udelay(16);
 	for (cs = 0; cs < 2; cs++) {
 		print_debug("Precharge CS");
 		print_debug_hex8(cs);
 		print_debug("\n");
 		write32(BAR+DCALADDR, 0x04000000);
 		write32(BAR+DCALCSR, (0x80000002 | ((cs+1)<<21)));
 		do data32 = read32(BAR+DCALCSR);
 		while (data32 & 0x80000000);
 	}

 	/* EMRS: Enable DLLs, set OCD calibration mode to default */
 	udelay(16);
 	for (cs = 0; cs < 2; cs++) {
 		print_debug("EMRS CS");
 		print_debug_hex8(cs);
 		print_debug("\n");
 		write32(BAR+DCALADDR, 0x0b840001);
 		write32(BAR+DCALCSR, (0x80000003 | ((cs+1)<<21)));
 		do data32 = read32(BAR+DCALCSR);
 		while (data32 & 0x80000000);
 	}
 	/* MRS: Reset DLLs */
 	udelay(16);
 	for (cs = 0; cs < 2; cs++) {
 		print_debug("MRS CS");
 		print_debug_hex8(cs);
 		print_debug("\n");
 		write32(BAR+DCALADDR, mode_reg);
 		write32(BAR+DCALCSR, (0x80000003 | ((cs+1)<<21)));
 		do data32 = read32(BAR+DCALCSR);
 		while (data32 & 0x80000000);
 	}

 	/* Precharge all banks */
 	udelay(48);
 	for (cs = 0; cs < 2; cs++) {
 		print_debug("Precharge CS");
 		print_debug_hex8(cs);
 		print_debug("\n");
 		write32(BAR+DCALADDR, 0x04000000);
 		write32(BAR+DCALCSR, (0x80000002 | ((cs+1)<<21)));
 		do data32 = read32(BAR+DCALCSR);
 		while (data32 & 0x80000000);
 	}

 	/* Do 2 refreshes */
 	for (i = 0; i < 2; i++) {
 		udelay(16);
 		for (cs = 0; cs < 2; cs++) {
 			print_debug("Refresh CS");
 			print_debug_hex8(cs);
 			print_debug("\n");
 			write32(BAR+DCALCSR, (0x80000001 | ((cs+1)<<21)));
 			do data32 = read32(BAR+DCALCSR);
 			while (data32 & 0x80000000);
 		}
 	}

 	/* MRS: Set DLLs to normal */
 	udelay(16);
 	for (cs = 0; cs < 2; cs++) {
 		print_debug("MRS CS");
 		print_debug_hex8(cs);
 		print_debug("\n");
 		write32(BAR+DCALADDR, (mode_reg & ~(1<<24)));
 		write32(BAR+DCALCSR, (0x80000003 | ((cs+1)<<21)));
 		do data32 = read32(BAR+DCALCSR);
 		while (data32 & 0x80000000);
 	}

 	/* EMRS: Enable DLLs */
 	udelay(16);
 	for (cs = 0; cs < 2; cs++) {
 		print_debug("EMRS CS");
 		print_debug_hex8(cs);
 		print_debug("\n");
 		write32(BAR+DCALADDR, 0x0b840001);
 		write32(BAR+DCALCSR, (0x80000003 | ((cs+1)<<21)));
 		do data32 = read32(BAR+DCALCSR);
 		while (data32 & 0x80000000);
         }

 	udelay(16);
 	/* No command */
 	write32(BAR+DCALCSR, 0x0000000f);

 	write32(BAR, 0x00100000);

 	/* Enable on-DIMM termination */
 	set_on_dimm_termination_enable(ctrl);

 	dump_dcal_regs();

 	/* Receive enable calibration */
 	udelay(16);
 	for (cs = 0; cs < 1; cs++) {
 		print_debug("receive enable calibration CS");
 		print_debug_hex8(cs);
 		print_debug("\n");
 		write32(BAR+DCALCSR, (0x8000000c | ((cs+1)<<21)));
 		do data32 = read32(BAR+DCALCSR);
 		while (data32 & 0x80000000);
 	}

 	dump_dcal_regs();

 	/* Adjust RCOMP */
 	data32 = read32(BAR+DDRIOMC2);
 	data32 &= ~(0xf << 16);
 	data32 |= (0xb << 16);
 	write32(BAR+DDRIOMC2, data32);

 	dump_dcal_regs();

 	data32 = drc & ~(3 << 20);  /* clear ECC mode */
 	pci_write_config32(ctrl->f0, DRC, data32);
 	write32(BAR+DCALCSR, 0x0008000f);

 	/* Clear memory and init ECC */
 	for (cs = 0; cs < 2; cs++) {
 		if (!(mask & (1<<cs)))
 			continue;
 		print_debug("clear memory CS");
 		print_debug_hex8(cs);
 		print_debug("\n");
 		write32(BAR+MBCSR, 0xa00000f0 | ((cs+1)<<20) | (0<<16));
 		do data32 = read32(BAR+MBCSR);
 		while (data32 & 0x80000000);
 		if (data32 & 0x40000000)
 			print_debug("failed!\n");
 	}

 	/* Clear read/write FIFO pointers */
 	print_debug("clear read/write fifo pointers\n");
 	write32(BAR+DDRIOMC2, read32(BAR+DDRIOMC2) | (1<<15));
 	udelay(16);
 	write32(BAR+DDRIOMC2, read32(BAR+DDRIOMC2) & ~(1<<15));
 	udelay(16);

 	dump_dcal_regs();

 	print_debug("Done\n");

 	/* Set initialization complete */
 	drc |= (1 << 29);
 	drc |= (3 << 30);
 	data32 = drc & ~(3 << 20);  /* clear ECC mode */
 	pci_write_config32(ctrl->f0, DRC, data32);

 	/* Set the ECC mode */
 	pci_write_config32(ctrl->f0, DRC, drc);

 	/* The memory is now set up--use it */
 	cache_ramstage();
 }

 static inline int memory_initialized(void)
 {
 	return pci_read_config32(PCI_DEV(0, 0x00, 0), DRC) & (1 << 29);
 }
	/*
	* This file is part of the coreboot project.
	*
	* Copyright (C) 2005 Eric W. Biederman and Tom Zimmerman
	* Copyright (C) 2008 Arastra, Inc.
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License version 2 as
	* published by the Free Software Foundation.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public License
	* along with this program; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <cpu/x86/mtrr.h>
	#include <cpu/x86/cache.h>
	#include "raminit_ep80579.h"
	#include "ep80579.h"

	#define BAR 0x90000000

	static void sdram_set_registers(const struct mem_controller *ctrl)
	{
	static const u32 register_values[] = {
	PCI_ADDR(0, 0x00, 0, CKDIS), 0xffff0000, 0x0000ffff,
	PCI_ADDR(0, 0x00, 0, DEVPRES), 0x00000000, 0x07420001 \| DEVPRES_CONFIG,
	PCI_ADDR(0, 0x00, 0, PAM-1), 0xcccccc7f, 0x33333000,
	PCI_ADDR(0, 0x00, 0, PAM+3), 0xcccccccc, 0x33333333,
	PCI_ADDR(0, 0x00, 0, DEVPRES1), 0xffffffff, 0x0040003a,
	PCI_ADDR(0, 0x00, 0, SMRBASE), 0x00000fff, BAR \| 0,
	};
	int i;
	int max;

	for (i = 0; i < ARRAY_SIZE(register_values); i += 3) {
	device_t dev;
	u32 where;
	u32 reg;
	dev = (register_values[i] & ~0xff) - PCI_DEV(0, 0x00, 0) + ctrl->f0;
	where = register_values[i] & 0xff;
	reg = pci_read_config32(dev, where);
	reg &= register_values[i+1];
	reg \|= register_values[i+2];
	pci_write_config32(dev, where, reg);
	}
	}

	struct dimm_size {
	u32 side1;
	u32 side2;
	};

	static struct dimm_size spd_get_dimm_size(u16 device)
	{
	/* Calculate the log base 2 size of a DIMM in bits */
	struct dimm_size sz;
	int value, low, ddr2;
	sz.side1 = 0;
	sz.side2 = 0;

	/* Note it might be easier to use byte 31 here, it has the DIMM size as
	* a multiple of 4MB. The way we do it now we can size both
	* sides of an assymetric dimm.
	*/
	value = spd_read_byte(device, SPD_NUM_ROWS);
	if (value < 0) goto hw_err;
	if ((value & 0xf) == 0) goto val_err;
	sz.side1 += value & 0xf;

	value = spd_read_byte(device, SPD_NUM_COLUMNS);
	if (value < 0) goto hw_err;
	if ((value & 0xf) == 0) goto val_err;
	sz.side1 += value & 0xf;

	value = spd_read_byte(device, SPD_NUM_BANKS_PER_SDRAM);
	if (value < 0) goto hw_err;
	if ((value & 0xff) == 0) goto val_err;
	sz.side1 += log2(value & 0xff);

	/* Get the module data width and convert it to a power of two */
	value = spd_read_byte(device, SPD_MODULE_DATA_WIDTH_MSB);
	if (value < 0) goto hw_err;
	value &= 0xff;
	value <<= 8;

	low = spd_read_byte(device, SPD_MODULE_DATA_WIDTH_LSB);
	if (low < 0) goto hw_err;
	value = value \| (low & 0xff);
	if ((value != 72) && (value != 64)) goto val_err;
	sz.side1 += log2(value);

	/* side 2 */
	value = spd_read_byte(device, SPD_NUM_DIMM_BANKS);

	if (value < 0) goto hw_err;
	value &= 7;
	value++;
	if (value == 1) goto out;
	if (value != 2) goto val_err;

	/* Start with the symmetrical case */
	sz.side2 = sz.side1;

	value = spd_read_byte(device, SPD_NUM_ROWS);
	if (value < 0) goto hw_err;
	if ((value & 0xf0) == 0) goto out; /* If symmetrical we are done */
	sz.side2 -= (value & 0x0f); /* Subtract out rows on side 1 */
	sz.side2 += ((value >> 4) & 0x0f); /* Add in rows on side 2 */

	value = spd_read_byte(device, SPD_NUM_COLUMNS);
	if (value < 0) goto hw_err;
	if ((value & 0xff) == 0) goto val_err;
	sz.side2 -= (value & 0x0f); /* Subtract out columns on side 1 */
	sz.side2 += ((value >> 4) & 0x0f); /* Add in columns on side 2 */
	goto out;

	val_err:
	die("Bad SPD value\n");
	/* If an hw_error occurs report that I have no memory */
	hw_err:
	sz.side1 = 0;
	sz.side2 = 0;
	out:
	print_debug("dimm ");
	print_debug_hex8(device);
	print_debug(" size = ");
	print_debug_hex8(sz.side1);
	print_debug(".");
	print_debug_hex8(sz.side2);
	print_debug("\n");
	return sz;

	}

	static long spd_set_ram_size(const struct mem_controller *ctrl, u8 dimm_mask)
	{
	int i;
	int cum;

	for (i = cum = 0; i < DIMM_SOCKETS; i++) {
	struct dimm_size sz;
	if (dimm_mask & (1 << i)) {
	sz = spd_get_dimm_size(ctrl->channel0[i]);
	if (sz.side1 < 29) {
	return -1; /* Report SPD error */
	}
	/* convert bits to multiples of 64MB */
	sz.side1 -= 29;
	cum += (1 << sz.side1);
	pci_write_config8(ctrl->f0, DRB + (i*2), cum);
	pci_write_config8(ctrl->f0, DRB+1 + (i*2), cum);
	if (spd_read_byte(ctrl->channel0[i], SPD_NUM_DIMM_BANKS) & 0x1) {
	cum <<= 1;
	}
	}
	else {
	pci_write_config8(ctrl->f0, DRB + (i*2), cum);
	pci_write_config8(ctrl->f0, DRB+1 + (i*2), cum);
	}
	}
	print_debug("DRB = ");
	print_debug_hex32(pci_read_config32(ctrl->f0, DRB));
	print_debug("\n");

	cum >>= 1;
	/* set TOM top of memory */
	pci_write_config16(ctrl->f0, TOM, cum);
	print_debug("TOM = ");
	print_debug_hex16(cum);
	print_debug("\n");
	/* set TOLM top of low memory */
	if (cum > 0x18) {
	cum = 0x18;
	}
	cum <<= 11;
	pci_write_config16(ctrl->f0, TOLM, cum);
	print_debug("TOLM = ");
	print_debug_hex16(cum);
	print_debug("\n");
	return 0;
	}


	static u8 spd_detect_dimms(const struct mem_controller *ctrl)
	{
	u8 dimm_mask = 0;
	int i;
	for (i = 0; i < DIMM_SOCKETS; i++) {
	int byte;
	u16 device;
	device = ctrl->channel0[i];
	if (device) {
	byte = spd_read_byte(device, SPD_MEMORY_TYPE);
	print_debug("spd ");
	print_debug_hex8(device);
	print_debug(" = ");
	print_debug_hex8(byte);
	print_debug("\n");
	if (byte == 8) {
	dimm_mask \|= (1 << i);
	}
	}
	}
	return dimm_mask;
	}


	static int spd_set_row_attributes(const struct mem_controller *ctrl,
	u8 dimm_mask)
	{
	int value;
	int i;

	for (i = 0; i < DIMM_SOCKETS; i++) {
	u32 dra = 0;
	int reg = 0;

	if (!(dimm_mask & (1 << i))) {
	continue;
	}

	value = spd_read_byte(ctrl->channel0[i], SPD_NUM_ROWS);
	if (value < 0) die("Bad SPD data\n");
	if ((value & 0xf) == 0) die("Invalid # of rows\n");
	dra \|= (((value-13) & 0x7) << 23);
	dra \|= (((value-13) & 0x7) << 29);
	reg += value & 0xf;

	value = spd_read_byte(ctrl->channel0[i], SPD_NUM_COLUMNS);
	if (value < 0) die("Bad SPD data\n");
	if ((value & 0xf) == 0) die("Invalid # of columns\n");
	dra \|= (((value-10) & 0x7) << 20);
	dra \|= (((value-10) & 0x7) << 26);
	reg += value & 0xf;

	value = spd_read_byte(ctrl->channel0[i], SPD_NUM_BANKS_PER_SDRAM);
	if (value < 0) die("Bad SPD data\n");
	if ((value & 0xff) == 0) die("Invalid # of banks\n");
	reg += log2(value & 0xff);

	print_debug("dimm ");
	print_debug_hex8(i);
	print_debug(" reg = ");
	print_debug_hex8(reg);
	print_debug("\n");

	/* set device density */
	dra \|= ((31-reg));
	dra \|= ((31-reg) << 6);

	/* set device width (x8) */
	dra \|= (1 << 4);
	dra \|= (1 << 10);

	/* set device type (registered) */
	dra \|= (1 << 14);

	/* set number of ranks (0=single, 1=dual) */
	value = spd_read_byte(ctrl->channel0[i], SPD_NUM_DIMM_BANKS);
	dra \|= ((value & 0x1) << 17);

	print_debug("DRA");
	print_debug_hex8(i);
	print_debug(" = ");
	print_debug_hex32(dra);
	print_debug("\n");

	pci_write_config32(ctrl->f0, DRA + (i*4), dra);
	}
	return 0;
	}


	static u32 spd_set_drt_attributes(const struct mem_controller *ctrl,
	u8 dimm_mask, u32 drc)
	{
	int i;
	u32 val, val1;
	u32 cl;
	u32 trc = 0;
	u32 trfc = 0;
	u32 tras = 0;
	u32 trtp = 0;
	u32 twtr = 0;
	int index = drc & 0x00000003;
	int ci;
	static const u8 latencies[] = { /* 533, 800, 400, 667 */
	0x10, 0x60, 0x10, 0x20 };
	static const u32 drt0[] = { /* 533, 800, 400, 667 */
	0x24240002, 0x24360002, 0x24220002, 0x24360002 };
	static const u32 drt1[] = { /* 533, 800, 400, 667 */
	0x00400000, 0x00900000, 0x00200000, 0x00700000 };
	static const u32 magic[] = { /* 533, 800, 400, 667 */
	0x007b8221, 0x00b94331, 0x005ca1a1, 0x009a62b1 };
	static const u32 mrs[] = { /* 533, 800, 400, 667 */
	0x07020000, 0x0b020000, 0x05020000, 0x09020000 };
	static const int cycle[] = { /* 533, 800, 400, 667 */
	15, 10, 20, 12 }; /* cycle time in 1/4 ns units */
	static const int byte40rem[] = {
	0, 1, 2, 2, 3, 3, 0, 0 }; /* byte 40 remainder in 1/4 ns units */

	/* CAS latency in cycles */
	val = latencies[index];
	for (i = 0; i < DIMM_SOCKETS; i++) {
	if (!(dimm_mask & (1 << i)))
	continue;
	val &= spd_read_byte(ctrl->channel0[i], SPD_ACCEPTABLE_CAS_LATENCIES);
	}
	if (val & 0x10)
	cl = 4;
	else if (val & 0x20)
	cl = 5;
	else if (val & 0x40)
	cl = 6;
	else
	die("CAS latency mismatch\n");
	print_debug("cl = ");
	print_debug_hex8(cl);
	print_debug("\n");

	ci = cycle[index];

	/* Trc, Trfc in cycles */
	for (i = 0; i < DIMM_SOCKETS; i++) {
	if (!(dimm_mask & (1 << i)))
	continue;
	val1 = spd_read_byte(ctrl->channel0[i], SPD_BYTE_41_42_EXTENSION);
	val = spd_read_byte(ctrl->channel0[i], SPD_MIN_ACT_TO_ACT_AUTO_REFRESH);
	val <<= 2; /* convert to 1/4 ns */
	val += byte40rem[(val1 >> 4) & 0x7];
	val = (val + ci - 1) / ci + 1; /* convert to cycles */
	if (trc < val)
	trc = val;
	val = spd_read_byte(ctrl->channel0[i], SPD_MIN_AUTO_REFRESH_TO_ACT);
	val <<= 2; /* convert to 1/4 ns */
	if (val1 & 0x01)
	val += 1024;
	val += byte40rem[(val1 >> 1) & 0x7];
	val = (val + ci - 1) / ci; /* convert to cycles */
	if (trfc < val)
	trfc = val;
	}
	print_debug("trc = ");
	print_debug_hex8(trc);
	print_debug("\n");
	print_debug("trfc = ");
	print_debug_hex8(trfc);
	print_debug("\n");

	/* Tras, Trtp, Twtr in cycles */
	for (i = 0; i < DIMM_SOCKETS; i++) {
	if (!(dimm_mask & (1 << i)))
	continue;
	val = spd_read_byte(ctrl->channel0[i], SPD_MIN_ACTIVE_TO_PRECHARGE_DELAY);
	val <<= 2; /* convert to 1/4 ns */
	val = (val + ci - 1) / ci; /* convert to cycles */
	if (tras < val)
	tras = val;
	val = spd_read_byte(ctrl->channel0[i], SPD_INT_READ_TO_PRECHARGE_DELAY);
	val = (val + ci - 1) / ci; /* convert to cycles */
	if (trtp < val)
	trtp = val;
	val = spd_read_byte(ctrl->channel0[i], SPD_INT_WRITE_TO_READ_DELAY);
	val = (val + ci - 1) / ci; /* convert to cycles */
	if (twtr < val)
	twtr = val;
	}
	print_debug("tras = ");
	print_debug_hex8(tras);
	print_debug("\n");
	print_debug("trtp = ");
	print_debug_hex8(trtp);
	print_debug("\n");
	print_debug("twtr = ");
	print_debug_hex8(twtr);
	print_debug("\n");

	val = (drt0[index] \| ((trc - 11) << 12) \| ((cl - 3) << 9)
	\| ((cl - 3) << 6) \| ((cl - 3) << 3));
	print_debug("drt0 = ");
	print_debug_hex32(val);
	print_debug("\n");
	pci_write_config32(ctrl->f0, DRT0, val);

	val = (drt1[index] \| ((tras - 8) << 28) \| ((trtp - 2) << 25)
	\| (twtr << 15));
	print_debug("drt1 = ");
	print_debug_hex32(val);
	print_debug("\n");
	pci_write_config32(ctrl->f0, DRT1, val);

	val = (magic[index]);
	print_debug("magic = ");
	print_debug_hex32(val);
	print_debug("\n");
	pci_write_config32(PCI_DEV(0, 0x08, 0), 0xcc, val);

	val = (mrs[index] \| (cl << 20));
	print_debug("mrs = ");
	print_debug_hex32(val);
	print_debug("\n");
	return val;
	}

	static int spd_set_dram_controller_mode(const struct mem_controller *ctrl,
	u8 dimm_mask)
	{
	int value;
	int drc = 0;
	int i;
	msr_t msr;
	u8 cycle = 0x25;

	for (i = 0; i < DIMM_SOCKETS; i++) {
	if (!(dimm_mask & (1 << i)))
	continue;
	if ((spd_read_byte(ctrl->channel0[i], SPD_MODULE_DATA_WIDTH_LSB) & 0xf0) != 0x40)
	die("ERROR: Only 64-bit DIMMs supported\n");
	if (!(spd_read_byte(ctrl->channel0[i], SPD_DIMM_CONFIG_TYPE) & 0x02))
	die("ERROR: Only ECC DIMMs supported\n");
	if (spd_read_byte(ctrl->channel0[i], SPD_PRIMARY_SDRAM_WIDTH) != 0x08)
	die("ERROR: Only x8 DIMMs supported\n");

	value = spd_read_byte(ctrl->channel0[i], SPD_MIN_CYCLE_TIME_AT_CAS_MAX);
	if (value > cycle)
	cycle = value;
	}
	print_debug("cycle = ");
	print_debug_hex8(cycle);
	print_debug("\n");

	drc \|= (1 << 20); /* enable ECC */
	drc \|= (3 << 30); /* enable CKE on each DIMM */
	drc \|= (1 << 4); /* enable CKE globally */

	/* TODO check: */
	/* set front side bus speed */
	msr = rdmsr(0xcd); /* returns 0 on Pentium M 90nm */
	print_debug("msr 0xcd = ");
	print_debug_hex32(msr.hi);
	print_debug_hex32(msr.lo);
	print_debug("\n");

	/* TODO check that this msr really indicates fsb speed! */
	if (msr.lo & 0x07) {
	print_info("533 MHz FSB\n");
	if (cycle <= 0x25) {
	drc \|= 0x5;
	print_info("400 MHz DDR\n");
	} else if (cycle <= 0x30) {
	drc \|= 0x7;
	print_info("333 MHz DDR\n");
	} else if (cycle <= 0x3d) {
	drc \|= 0x4;
	print_info("266 MHz DDR\n");
	} else {
	drc \|= 0x2;
	print_info("200 MHz DDR\n");
	}
	}
	else {
	print_info("400 MHz FSB\n");
	if (cycle <= 0x30) {
	drc \|= 0x7;
	print_info("333 MHz DDR\n");
	} else if (cycle <= 0x3d) {
	drc \|= 0x0;
	print_info("266 MHz DDR\n");
	} else {
	drc \|= 0x2;
	print_info("200 MHz DDR\n");
	}
	}

	print_debug("DRC = ");
	print_debug_hex32(drc);
	print_debug("\n");

	return drc;
	}

	static void sdram_set_spd_registers(const struct mem_controller *ctrl)
	{
	u8 dimm_mask;
	int i;

	/* Test if we can read the SPD */
	dimm_mask = spd_detect_dimms(ctrl);
	if (!(dimm_mask & ((1 << DIMM_SOCKETS) - 1))) {
	print_err("No memory for this cpu\n");
	return;
	}
	return;
	}

	static void set_on_dimm_termination_enable(const struct mem_controller *ctrl)
	{
	u8 c1,c2;
	u32 dimm,i;
	u32 data32;
	u32 t4;

	/* Set up northbridge values */
	/* ODT enable */
	pci_write_config32(ctrl->f0, SDRC, 0xa0002c30);

	c1 = pci_read_config8(ctrl->f0, DRB);
	c2 = pci_read_config8(ctrl->f0, DRB+2);
	if (c1 == c2) {
	/* 1 single-rank DIMM */
	data32 = 0x00000010;
	}
	else {
	/* 2 single-rank DIMMs or 1 double-rank DIMM */
	data32 = 0x00002010;
	}

	print_debug("ODT Value = ");
	print_debug_hex32(data32);
	print_debug("\n");

	pci_write_config32(ctrl->f0, DDR2ODTC, data32);

	for (i = 0; i < 2; i++) {
	print_debug("ODT CS");
	print_debug_hex8(i);
	print_debug("\n");

	write32(BAR+DCALADDR, 0x0b840001);
	write32(BAR+DCALCSR, 0x80000003 \| ((i+1)<<21));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}
	}


	static void dump_dcal_regs(void)
	{
	int i;
	for (i = 0x0; i < 0x2a0; i += 4) {
	if ((i % 16) == 0) {
	print_debug("\n");
	print_debug_hex16(i);
	print_debug(": ");
	}
	print_debug_hex32(read32(BAR+i));
	print_debug(" ");
	}
	print_debug("\n");
	}


	static void sdram_enable(int controllers, const struct mem_controller *ctrl)
	{
	int i;
	int cs;
	long mask;
	u32 drc;
	u32 data32;
	u32 mode_reg;
	msr_t msr;
	u16 data16;

	mask = spd_detect_dimms(ctrl);
	print_debug("Starting SDRAM Enable\n");

	/* Set DRAM type and Front Side Bus frequency */
	drc = spd_set_dram_controller_mode(ctrl, mask);
	if (drc == 0) {
	die("Error calculating DRC\n");
	}
	data32 = drc & ~(3 << 20); /* clear ECC mode */
	data32 = data32 \| (3 << 5); /* temp turn off ODT */
	/* Set DRAM controller mode */
	pci_write_config32(ctrl->f0, DRC, data32);

	/* Turn the clocks on */
	pci_write_config16(ctrl->f0, CKDIS, 0x0000);

	/* Program row size */
	spd_set_ram_size(ctrl, mask);

	/* Program row attributes */
	spd_set_row_attributes(ctrl, mask);

	/* Program timing values */
	mode_reg = spd_set_drt_attributes(ctrl, mask, drc);

	dump_dcal_regs();

	/* Apply NOP */
	for (cs = 0; cs < 2; cs++) {
	print_debug("NOP CS");
	print_debug_hex8(cs);
	print_debug("\n");
	udelay(16);
	write32(BAR+DCALCSR, (0x00000000 \| ((cs+1)<<21)));
	write32(BAR+DCALCSR, (0x80000000 \| ((cs+1)<<21)));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}

	/* Apply NOP */
	udelay(16);
	for (cs = 0; cs < 2; cs++) {
	print_debug("NOP CS");
	print_debug_hex8(cs);
	print_debug("\n");
	write32(BAR + DCALCSR, (0x80000000 \| ((cs+1)<<21)));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}

	/* Precharge all banks */
	udelay(16);
	for (cs = 0; cs < 2; cs++) {
	print_debug("Precharge CS");
	print_debug_hex8(cs);
	print_debug("\n");
	write32(BAR+DCALADDR, 0x04000000);
	write32(BAR+DCALCSR, (0x80000002 \| ((cs+1)<<21)));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}

	/* EMRS: Enable DLLs, set OCD calibration mode to default */
	udelay(16);
	for (cs = 0; cs < 2; cs++) {
	print_debug("EMRS CS");
	print_debug_hex8(cs);
	print_debug("\n");
	write32(BAR+DCALADDR, 0x0b840001);
	write32(BAR+DCALCSR, (0x80000003 \| ((cs+1)<<21)));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}
	/* MRS: Reset DLLs */
	udelay(16);
	for (cs = 0; cs < 2; cs++) {
	print_debug("MRS CS");
	print_debug_hex8(cs);
	print_debug("\n");
	write32(BAR+DCALADDR, mode_reg);
	write32(BAR+DCALCSR, (0x80000003 \| ((cs+1)<<21)));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}

	/* Precharge all banks */
	udelay(48);
	for (cs = 0; cs < 2; cs++) {
	print_debug("Precharge CS");
	print_debug_hex8(cs);
	print_debug("\n");
	write32(BAR+DCALADDR, 0x04000000);
	write32(BAR+DCALCSR, (0x80000002 \| ((cs+1)<<21)));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}

	/* Do 2 refreshes */
	for (i = 0; i < 2; i++) {
	udelay(16);
	for (cs = 0; cs < 2; cs++) {
	print_debug("Refresh CS");
	print_debug_hex8(cs);
	print_debug("\n");
	write32(BAR+DCALCSR, (0x80000001 \| ((cs+1)<<21)));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}
	}

	/* MRS: Set DLLs to normal */
	udelay(16);
	for (cs = 0; cs < 2; cs++) {
	print_debug("MRS CS");
	print_debug_hex8(cs);
	print_debug("\n");
	write32(BAR+DCALADDR, (mode_reg & ~(1<<24)));
	write32(BAR+DCALCSR, (0x80000003 \| ((cs+1)<<21)));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}

	/* EMRS: Enable DLLs */
	udelay(16);
	for (cs = 0; cs < 2; cs++) {
	print_debug("EMRS CS");
	print_debug_hex8(cs);
	print_debug("\n");
	write32(BAR+DCALADDR, 0x0b840001);
	write32(BAR+DCALCSR, (0x80000003 \| ((cs+1)<<21)));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}

	udelay(16);
	/* No command */
	write32(BAR+DCALCSR, 0x0000000f);

	write32(BAR, 0x00100000);

	/* Enable on-DIMM termination */
	set_on_dimm_termination_enable(ctrl);

	dump_dcal_regs();

	/* Receive enable calibration */
	udelay(16);
	for (cs = 0; cs < 1; cs++) {
	print_debug("receive enable calibration CS");
	print_debug_hex8(cs);
	print_debug("\n");
	write32(BAR+DCALCSR, (0x8000000c \| ((cs+1)<<21)));
	do data32 = read32(BAR+DCALCSR);
	while (data32 & 0x80000000);
	}

	dump_dcal_regs();

	/* Adjust RCOMP */
	data32 = read32(BAR+DDRIOMC2);
	data32 &= ~(0xf << 16);
	data32 \|= (0xb << 16);
	write32(BAR+DDRIOMC2, data32);

	dump_dcal_regs();

	data32 = drc & ~(3 << 20); /* clear ECC mode */
	pci_write_config32(ctrl->f0, DRC, data32);
	write32(BAR+DCALCSR, 0x0008000f);

	/* Clear memory and init ECC */
	for (cs = 0; cs < 2; cs++) {
	if (!(mask & (1<<cs)))
	continue;
	print_debug("clear memory CS");
	print_debug_hex8(cs);
	print_debug("\n");
	write32(BAR+MBCSR, 0xa00000f0 \| ((cs+1)<<20) \| (0<<16));
	do data32 = read32(BAR+MBCSR);
	while (data32 & 0x80000000);
	if (data32 & 0x40000000)
	print_debug("failed!\n");
	}

	/* Clear read/write FIFO pointers */
	print_debug("clear read/write fifo pointers\n");
	write32(BAR+DDRIOMC2, read32(BAR+DDRIOMC2) \| (1<<15));
	udelay(16);
	write32(BAR+DDRIOMC2, read32(BAR+DDRIOMC2) & ~(1<<15));
	udelay(16);

	dump_dcal_regs();

	print_debug("Done\n");

	/* Set initialization complete */
	drc \|= (1 << 29);
	drc \|= (3 << 30);
	data32 = drc & ~(3 << 20); /* clear ECC mode */
	pci_write_config32(ctrl->f0, DRC, data32);

	/* Set the ECC mode */
	pci_write_config32(ctrl->f0, DRC, drc);

	/* The memory is now set up--use it */
	cache_ramstage();
	}

	static inline int memory_initialized(void)
	{
	return pci_read_config32(PCI_DEV(0, 0x00, 0), DRC) & (1 << 29);
	}