Below is a patch for sys/dev/ic/fxp.c that fixes random pool corruption
when ioctl()s SIOCSIFADDR, SIOCSIFFLAGS, SIOCADDMULTI, SIOCDELMULTI, take
place on an fxp interface that is receiving packets.  It also fixes a
potential panic at ifdown time due to uninitialized pointer usage
resulting in random dmabuf unmapping.

The current driver partially resets the chip when these IOCTLs takes
place, and reinitialises a register on the chip to the start of
(effectively) the current "rx mbufs" chain with no regard to whether these
mbufs were already marked as ready to be processed or not.  When the chip
then received another packet (potentially some time later) the first mbuf
would be overwritten and an interrupt would trigger rx processing.  The
driver then processes the whole chain, handing the mbufs off to
ether_input_mbuf(), which processes the mbufs received before the reset
and frees them...  This is where it goes bad, because the chip still has
pointers to those mbufs and keeps filling them up.
  
This was exhibiting itself as corruption of the mbuf cluster pool, the
pool corruption not being detected by DIAGNOSTIC checks because the status
gets written at offset 4 into the pool pages - instead it silently trashes
the TAILQs and you get a panic quite some time later.
  
This is probably what has caused me random panics on a regular basis since
OPENBSD_2_9 days, when we switched from xl's to fxp's as they were the
recommended NICs and the xl's were the suggested cause of all my
problems....  that was just the beginning :)
  
Of course, nothing was repeatable - until now.  Over the last few days,
David(dlg) wanted me to compare networking performance of his new Athlon64
desktop to our P4's, so I was forced to move pfsync back to a fxp
interface, and it turns out that having pfsync enabled (or alternatively,
ping flooding) causes enough traffic around the time that these ioctls
take place to crash a system reliably within a few seconds.
  
Blabber aside, on to what the patch does - it uses the chip's software
generated interrupt facility to queue a hardware IRQ, which we then catch
and use to reprogram the RX queue registers (after processing any
already-received packets) rather than blindly resetting it in the fxp_init
routine to a potentially bad value.

Chris Pascoe

Index: fxp.c
===================================================================
RCS file: /cvs/src/sys/dev/ic/fxp.c,v
retrieving revision 1.60
diff -u -p -r1.60 fxp.c
--- fxp.c	7 Nov 2004 01:13:48 -0000	1.60
+++ fxp.c	3 Dec 2004 09:47:59 -0000
@@ -873,7 +873,8 @@ fxp_intr(arg)
 
 	while ((statack = CSR_READ_1(sc, FXP_CSR_SCB_STATACK)) != 0) {
 		claimed = 1;
-		rnr = (statack & FXP_SCB_STATACK_RNR) ? 1 : 0;
+		rnr = (statack & (FXP_SCB_STATACK_RNR | 
+		                  FXP_SCB_STATACK_SWI)) ? 1 : 0;
 		/*
 		 * First ACK all the interrupts in this pass.
 		 */
@@ -922,7 +923,8 @@ fxp_intr(arg)
 		 * not ready (RNR) condition exists, get whatever
 		 * packets we can and re-start the receiver.
 		 */
-		if (statack & (FXP_SCB_STATACK_FR | FXP_SCB_STATACK_RNR)) {
+		if (statack & (FXP_SCB_STATACK_FR | FXP_SCB_STATACK_RNR |
+			       FXP_SCB_STATACK_SWI)) {
 			struct mbuf *m;
 			u_int8_t *rfap;
 rcvloop:
@@ -1206,7 +1208,6 @@ fxp_init(xsc)
 	struct fxp_cb_config *cbp;
 	struct fxp_cb_ias *cb_ias;
 	struct fxp_cb_tx *txp;
-	struct mbuf *m;
 	bus_dmamap_t rxmap;
 	int i, prm, allm, s, bufs;
 
@@ -1407,10 +1408,10 @@ fxp_init(xsc)
 		bufs = FXP_NRFABUFS_MIN;
 	if (sc->rx_bufs > bufs) {
 		while (sc->rfa_headm != NULL && sc->rx_bufs-- > bufs) {
-			rxmap = *((bus_dmamap_t *)m->m_ext.ext_buf);
+			rxmap = *((bus_dmamap_t *)sc->rfa_headm->m_ext.ext_buf);
 			bus_dmamap_unload(sc->sc_dmat, rxmap);
 			FXP_RXMAP_PUT(sc, rxmap);
-			sc->rfa_headm = m_free(m);
+			sc->rfa_headm = m_free(sc->rfa_headm);
 		}
 	} else if (sc->rx_bufs < bufs) {
 		int err, tmp_rx_bufs = sc->rx_bufs;
@@ -1428,10 +1429,6 @@ fxp_init(xsc)
 				break;
 	}
 	fxp_scb_wait(sc);
-	rxmap = *((bus_dmamap_t *)sc->rfa_headm->m_ext.ext_buf);
-	CSR_WRITE_4(sc, FXP_CSR_SCB_GENERAL,
-	    rxmap->dm_segs[0].ds_addr + RFA_ALIGNMENT_FUDGE);
-	fxp_scb_cmd(sc, FXP_SCB_COMMAND_RU_START);
 
 	/*
 	 * Set current media.
@@ -1440,6 +1437,16 @@ fxp_init(xsc)
 
 	ifp->if_flags |= IFF_RUNNING;
 	ifp->if_flags &= ~IFF_OACTIVE;
+
+	/*
+	 * Request a software generated interrupt that will be used to
+	 * restart the RX queue.  If we just direct the chip to start
+	 * receiving from the start of queue instead of having the 
+	 * interrupt handler process all received packets, we run the
+	 * risk of having it overwrite mbuf clusters while they are 
+	 * being processed or after they have been returned to the pool.
+	 */
+	CSR_WRITE_1(sc, FXP_CSR_SCB_INTRCNTL, FXP_SCB_INTRCNTL_REQUEST_SWI);
 	splx(s);
 
 	/*
Index: fxpreg.h
===================================================================
RCS file: /cvs/src/sys/dev/ic/fxpreg.h,v
retrieving revision 1.6
diff -u -r1.6 fxpreg.h
--- fxpreg.h	4 Aug 2004 19:42:30 -0000	1.6
+++ fxpreg.h	3 Dec 2004 09:48:09 -0000
@@ -96,6 +96,8 @@
 #define FXP_SCB_COMMAND_RU_BASE		6
 #define FXP_SCB_COMMAND_RU_RBDRESUME	7
 
+#define FXP_SCB_INTRCNTL_REQUEST_SWI	0x02
+
 /*
  * Command block definitions
  */