Add two functions xpt_batch_start() and xpt_batch_done() to the CAM SIM KPI

to allow drivers to handle request completion directly without passing
them to the CAM SWI thread removing extra context switch.
Modify all ATA/SATA drivers to use them.

Reviewed by:	gibbs, ken
MFC after:	2 weeks

sys/cam/cam_sim.h

@@ -106,6 +106,7 @@ struct cam_sim {
 #define	CAM_SIM_MPSAFE			0x02
 #define	CAM_SIM_ON_DONEQ		0x04
 #define	CAM_SIM_POLLED			0x08
+#define	CAM_SIM_BATCH			0x10
 	struct callout		callout;
 	struct cam_devq 	*devq;	/* Device Queue to use for this SIM */
 	int			refcount; /* References to the SIM. */

sys/cam/cam_xpt.c

@@ -4332,7 +4332,8 @@ xpt_done(union ccb *done_ccb)
 		TAILQ_INSERT_TAIL(&sim->sim_doneq, &done_ccb->ccb_h,
 		    sim_links.tqe);
 		done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
-		if ((sim->flags & (CAM_SIM_ON_DONEQ | CAM_SIM_POLLED)) == 0) {
+		if ((sim->flags & (CAM_SIM_ON_DONEQ | CAM_SIM_POLLED |
+		    CAM_SIM_BATCH)) == 0) {
 			mtx_lock(&cam_simq_lock);
 			first = TAILQ_EMPTY(&cam_simq);
 			TAILQ_INSERT_TAIL(&cam_simq, sim, links);
@@ -4344,6 +4345,25 @@ xpt_done(union ccb *done_ccb)
 	}
 }
 
+void
+xpt_batch_start(struct cam_sim *sim)
+{
+
+	KASSERT((sim->flags & CAM_SIM_BATCH) == 0, ("Batch flag already set"));
+	sim->flags |= CAM_SIM_BATCH;
+}
+
+void
+xpt_batch_done(struct cam_sim *sim)
+{
+
+	KASSERT((sim->flags & CAM_SIM_BATCH) != 0, ("Batch flag was not set"));
+	sim->flags &= ~CAM_SIM_BATCH;
+	if (!TAILQ_EMPTY(&sim->sim_doneq) &&
+	    (sim->flags & CAM_SIM_ON_DONEQ) == 0)
+		camisr_runqueue(&sim->sim_doneq);
+}
+
 union ccb *
 xpt_alloc_ccb()
 {

sys/cam/cam_xpt_sim.h

@@ -51,6 +51,8 @@ void		xpt_release_devq_rl(struct cam_path *path, cam_rl rl,
 		    u_int count, int run_queue);
 int		xpt_sim_opened(struct cam_sim *sim);
 void		xpt_done(union ccb *done_ccb);
+void		xpt_batch_start(struct cam_sim *sim);
+void		xpt_batch_done(struct cam_sim *sim);
 #endif
 
 #endif /* _CAM_CAM_XPT_SIM_H */

sys/dev/ahci/ahci.c

@@ -1457,7 +1457,9 @@ ahci_ch_intr_locked(void *data)
 	struct ahci_channel *ch = device_get_softc(dev);
 
 	mtx_lock(&ch->mtx);
+	xpt_batch_start(ch->sim);
 	ahci_ch_intr(data);
+	xpt_batch_done(ch->sim);
 	mtx_unlock(&ch->mtx);
 }
 

sys/dev/ata/ata-all.c

@@ -544,9 +544,11 @@ ata_interrupt(void *data)
     struct ata_channel *ch = (struct ata_channel *)data;
 
     mtx_lock(&ch->state_mtx);
+    xpt_batch_start(ch->sim);
 #endif
     ata_interrupt_locked(data);
 #ifdef ATA_CAM
+    xpt_batch_done(ch->sim);
     mtx_unlock(&ch->state_mtx);
 #endif
 }

sys/dev/mvs/mvs.c

@@ -654,7 +654,9 @@ mvs_ch_intr_locked(void *data)
 	struct mvs_channel *ch = device_get_softc(dev);
 
 	mtx_lock(&ch->mtx);
+	xpt_batch_start(ch->sim);
 	mvs_ch_intr(data);
+	xpt_batch_done(ch->sim);
 	mtx_unlock(&ch->mtx);
 }
 

sys/dev/siis/siis.c

@@ -830,7 +830,9 @@ siis_ch_intr_locked(void *data)
 	struct siis_channel *ch = device_get_softc(dev);
 
 	mtx_lock(&ch->mtx);
+	xpt_batch_start(ch->sim);
 	siis_ch_intr(data);
+	xpt_batch_done(ch->sim);
 	mtx_unlock(&ch->mtx);
 }