separate statclock(9) with mips3 CP0 timer interrupt

Hi,

Current several mips ports with MIP3 CPUs (evbmips, sgimips
etc.)
use CPU internal CP0 clock timer interrupts for
hardclock(9),
but some other ports (arc, cobalt etc) use their own
external
timer interrupts for hardclock(9) and CPU CP0 clock
interrupt
(i.e. CPU INT5) is not used.

Is it worth to prepare a separate interrupt handler for
statclock(9) with CP0 clock interrupt on such mips port?
I guess it may provide more precise statistics on the
system,
but I'm afraid number of interrupts (and its overhead) is
also
increased ~two times.

I've written some code for NetBSD/cobalt (derived from
hp300 and
arm/footbridge), but is there any good benchmark for this?
---
Izumi Tsutsui


Index: cobalt/autoconf.c
============================================================
=======
RCS file: /cvsroot/src/sys/arch/cobalt/cobalt/autoconf.c,v
retrieving revision 1.19
diff -u -r1.19 autoconf.c
--- cobalt/autoconf.c	21 Apr 2006 18:17:45 -0000	1.19
+++ cobalt/autoconf.c	22 Apr 2006 01:21:48 -0000
 -52,11
+52,13 
 	(void)splhigh();
 
 	evcnt_attach_static(&hardclock_ev);
+	evcnt_attach_static(&statclock_ev);
 
 	if (config_rootfound("mainbus", NULL) == NULL)
 		panic("no mainbus found");
 
-	_splnone();
+	/* turn on interrupts except cpu clock */
+	_spllower(MIPS_INT_MASK_5);
 }
 
 void
Index: cobalt/clock.c
============================================================
=======
RCS file: /cvsroot/src/sys/arch/cobalt/cobalt/clock.c,v
retrieving revision 1.12
diff -u -r1.12 clock.c
--- cobalt/clock.c	21 Apr 2006 16:52:15 -0000	1.12
+++ cobalt/clock.c	22 Apr 2006 01:21:48 -0000
 -47,6
+47,20 
 void *timer_cookie;
 
 /*
+ * Statistics clock variance, in usec.  Variance must be a
+ * power of two.  Since this gives us an even number, not
an odd number,
+ * we discard one case and compensate.  That is, a variance
of 1024 would
+ * give us offsets in [0..1023].  Instead, we take offsets
in [1..1023].
+ * This is symmetric about the point 512, or statvar/2, and
thus averages
+ * to that value (assuming uniform random numbers).
+ */
+static const uint32_t statvar = 1024;
+static uint32_t statint;	/* number of clock ticks for
stathz */
+static uint32_t statmin;	/* minimum stat clock count in
ticks */
+static uint32_t statprev;/* last value of we set statclock
to */
+static u_int statcountperusec;	/* number of ticks per usec
at current stathz */
+
+/*
  * Common parts of todclock autoconfiguration.
  */
 void
 -63,12
+77,21 
 cpu_initclocks(void)
 {
 
-	/* start timer */
+	if (stathz == 0)
+		stathz = hz;
+
+	if (profhz == 0)
+		profhz = hz * 5;
+
+	setstatclockrate(stathz);
+
+	/* start timer interrups for hardclock */
 	if (timer_start == NULL)
 		panic("cpu_initclocks(): no timer
configured");
 	(*timer_start)(timer_cookie);
 
-	return;
+	/* enable statclock intr (CPU INT5) */
+	_splnone();
 }
 
 /*
 -139,11
+162,59 
 }
 
 void
-setstatclockrate(int arg)
+setstatclockrate(int newhz)
 {
-	/* XXX */
+	uint32_t countpersecond, statvarticks;
+
+	statprev = mips3_cp0_count_read();
+
+	statint = ((curcpu()->ci_cpu_freq + newhz / 2) / newhz)
/ 2;
+
+	/* Get the total ticks a second */
+	countpersecond = statint * newhz;
+
+	/* now work out how many ticks per usec */
+	statcountperusec = countpersecond / 1000000;
+
+	/* calculate a variance range of statvar */
+	statvarticks = statcountperusec * statvar;
+
+	/* minimum is statint - 50% of variant */
+	statmin = statint - (statvarticks / 2);
+
+	mips3_cp0_compare_write(statprev + statint);
+}
+
+void
+statclockintr(struct clockframe *cfp)
+{
+	uint32_t curcount, statnext, delta, r;
+	int lost;
+
+	lost = 0;
+
+	do {
+		r = (uint32_t)random() & (statvar - 1);
+	} while (r == 0);
+	statnext = statprev + statmin + (r * statcountperusec);
+
+	mips3_cp0_compare_write(statnext);
+	curcount = mips3_cp0_count_read();
+	delta = statnext - curcount;
+
+	while ((int32_t)delta < 0) {
+		lost++;
+		delta += statint;
+	}
+	if (lost > 0) {
+		statnext = curcount + delta;
+		mips3_cp0_compare_write(statnext);
+		for (; lost > 0; lost--)
+			statclock(cfp);
+	}
+	statclock(cfp);
 
-	return;
+	statprev = statnext;
 }
 
 void
Index: cobalt/clockvar.h
============================================================
=======
RCS file: /cvsroot/src/sys/arch/cobalt/cobalt/clockvar.h,v
retrieving revision 1.3
diff -u -r1.3 clockvar.h
--- cobalt/clockvar.h	21 Apr 2006 18:17:45 -0000	1.3
+++ cobalt/clockvar.h	22 Apr 2006 01:21:48 -0000
 -25,7
+25,10 
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.
  */
 
+void statclockintr(struct clockframe *);
+
 extern struct evcnt hardclock_ev;
+extern struct evcnt statclock_ev;
 
 extern void (*timer_start)(void *);
 extern long (*timer_read)(void *);
Index: cobalt/machdep.c
============================================================
=======
RCS file: /cvsroot/src/sys/arch/cobalt/cobalt/machdep.c,v
retrieving revision 1.66
diff -u -r1.66 machdep.c
--- cobalt/machdep.c	21 Apr 2006 18:21:30 -0000	1.66
+++ cobalt/machdep.c	22 Apr 2006 01:21:48 -0000
 -110,6
+110,9 
 
 struct evcnt hardclock_ev =
     EVCNT_INITIALIZER(EVCNT_TYPE_INTR, NULL,
"cpu", "hardclock");
+struct evcnt statclock_ev =
+    EVCNT_INITIALIZER(EVCNT_TYPE_INTR, NULL,
"cpu", "statclock");
+
 
 u_int cobalt_id;
 static const char * const cobalt_model[] =
 -451,11
+454,22 
 cpu_intr(uint32_t status, uint32_t cause, uint32_t pc,
uint32_t ipending)
 {
 	struct clockframe cf;
-	static uint32_t cycles;
 	struct cobalt_intrhand *ih;
 
 	uvmexp.intrs++;
 
+	if (ipending & MIPS_INT_MASK_5) {
+		/* statclock */
+		cf.pc = pc;
+		cf.sr = status;
+
+		statclockintr(&cf);
+		statclock_ev.ev_count++;
+
+		cause &= ~MIPS_INT_MASK_5;
+	}
+	_splset((status & MIPS_INT_MASK_5) | MIPS_SR_INT_IE);
+
 	if (ipending & MIPS_INT_MASK_0) {
 		/* GT64x11 timer0 for hardclock */
 		volatile uint32_t *irq_src =
 -467,6
+481,29 
 			cf.pc = pc;
 			cf.sr = status;
 
+			if ((status & MIPS_INT_MASK) == MIPS_INT_MASK) {
+				if ((ipending & MIPS_INT_MASK &
+				     ~MIPS_INT_MASK_0) == 0) {
+					/*
+					 * If all interrupts were enabled and
+					 * there is no pending interrupts,
+					 * set MIPS_SR_INT_IE so that
+					 * spllowerclock() in hardclock()
+					 * works properly.
+					 */
+#if 0					/* MIPS_SR_INT_IE is enabled above */
+					_splset(MIPS_SR_INT_IE);
+#endif
+ 				} else {
+					/*
+					 * If there are any pending interrputs,
+					 * clear MIPS_SR_INT_IE in cf.sr so that
+					 * spllowerclock() in hardclock() will
+					 * not happen.
+					 */
+					cf.sr &= ~MIPS_SR_INT_IE;
+				}
+			}
 			hardclock(&cf);
 			hardclock_ev.ev_count++;
 		}
 -474,20
+511,6 
 	}
 	_splset((status & ~cause & MIPS_HARD_INT_MASK) |
MIPS_SR_INT_IE);
 
-	if (ipending & MIPS_INT_MASK_5) {
-		cycles = mips3_cp0_count_read();
-		mips3_cp0_compare_write(cycles + 1250000);	/* XXX */
-
-#if 0
-		cf.pc = pc;
-		cf.sr = status;
-
-		statclock(&cf);
-#endif
-		cause &= ~MIPS_INT_MASK_5;
-	}
-	_splset((status & ~cause & MIPS_HARD_INT_MASK) |
MIPS_SR_INT_IE);
-
 	if (ipending & MIPS_INT_MASK_3) {
 		/* 16650 serial */
 		ih = &intrtab[3];
Index: conf/std.cobalt
============================================================
=======
RCS file: /cvsroot/src/sys/arch/cobalt/conf/std.cobalt,v
retrieving revision 1.12
diff -u -r1.12 std.cobalt
--- conf/std.cobalt	11 Dec 2005 12:17:06 -0000	1.12
+++ conf/std.cobalt	22 Apr 2006 01:21:48 -0000
 -5,6
+5,7 
 makeoptions	MACHINE_ARCH="mipsel"
 
 options 	MIPS3
+options 	MIPS3_ENABLE_CLOCK_INTR
 
 options 	EXEC_ELF32	# exec ELF32 binaries
 options 	EXEC_SCRIPT	# exec #! scripts
Index: include/intr.h
============================================================
=======
RCS file: /cvsroot/src/sys/arch/cobalt/include/intr.h,v
retrieving revision 1.20
diff -u -r1.20 intr.h
--- include/intr.h	21 Apr 2006 18:17:45 -0000	1.20
+++ include/intr.h	22 Apr 2006 01:21:48 -0000
 -80,7
+80,8 
 #define SPLBIO		(SPLSOFT | MIPS_INT_MASK_4)
 #define SPLNET		(SPLBIO | MIPS_INT_MASK_1 |
MIPS_INT_MASK_2)
 #define SPLTTY		(SPLNET | MIPS_INT_MASK_3)
-#define SPLCLOCK	(SPLTTY | MIPS_INT_MASK_0 |
MIPS_INT_MASK_5)
+#define SPLCLOCK	(SPLTTY | MIPS_INT_MASK_0)
+#define SPLSTATCLOCK	(SPLCLOCK | MIPS_INT_MASK_5)
 #define splbio()	_splraise(SPLBIO)
 #define splnet()	_splraise(SPLNET)
 #define spltty()	_splraise(SPLTTY)
 -88,7
+89,7 
 #define splserial()	_splraise(SPLTTY)
 #define splclock()	_splraise(SPLCLOCK)
 #define splvm()		splclock()
-#define splstatclock()	splclock()
+#define splstatclock()	_splraise(SPLSTATCLOCK)
 #define spllowersoftclock() _spllower(MIPS_SOFT_INT_MASK_0)
 
 #define	splsched()	splhigh()

Out of curiosity, why not use CPU INT5 (CP0 clock interrupt)
for
hardclock? 

I never even knew there was support for a separate
statistics clock.

    -- Garrett

Izumi Tsutsui wrote:
> Hi,
>
> Current several mips ports with MIP3 CPUs (evbmips,
sgimips etc.)
> use CPU internal CP0 clock timer interrupts for
hardclock(9),
> but some other ports (arc, cobalt etc) use their own
external
> timer interrupts for hardclock(9) and CPU CP0 clock
interrupt
> (i.e. CPU INT5) is not used.
>
> Is it worth to prepare a separate interrupt handler for
> statclock(9) with CP0 clock interrupt on such mips
port?
> I guess it may provide more precise statistics on the
system,
> but I'm afraid number of interrupts (and its overhead)
is also
> increased ~two times.
>
> I've written some code for NetBSD/cobalt (derived from
hp300 and
> arm/footbridge), but is there any good benchmark for
this?
> ---
> Izumi Tsutsui
>
>
> Index: cobalt/autoconf.c
>
============================================================
=======
> RCS file:
/cvsroot/src/sys/arch/cobalt/cobalt/autoconf.c,v
> retrieving revision 1.19
> diff -u -r1.19 autoconf.c
> --- cobalt/autoconf.c	21 Apr 2006 18:17:45 -0000	1.19
> +++ cobalt/autoconf.c	22 Apr 2006 01:21:48 -0000
>  -52,11 +52,13 
>  	(void)splhigh();
>  
>  	evcnt_attach_static(&hardclock_ev);
> +	evcnt_attach_static(&statclock_ev);
>  
>  	if (config_rootfound("mainbus", NULL) ==
NULL)
>  		panic("no mainbus found");
>  
> -	_splnone();
> +	/* turn on interrupts except cpu clock */
> +	_spllower(MIPS_INT_MASK_5);
>  }
>  
>  void
> Index: cobalt/clock.c
>
============================================================
=======
> RCS file: /cvsroot/src/sys/arch/cobalt/cobalt/clock.c,v
> retrieving revision 1.12
> diff -u -r1.12 clock.c
> --- cobalt/clock.c	21 Apr 2006 16:52:15 -0000	1.12
> +++ cobalt/clock.c	22 Apr 2006 01:21:48 -0000
>  -47,6 +47,20 
>  void *timer_cookie;
>  
>  /*
> + * Statistics clock variance, in usec.  Variance must
be a
> + * power of two.  Since this gives us an even number,
not an odd number,
> + * we discard one case and compensate.  That is, a
variance of 1024 would
> + * give us offsets in [0..1023].  Instead, we take
offsets in [1..1023].
> + * This is symmetric about the point 512, or
statvar/2, and thus averages
> + * to that value (assuming uniform random numbers).
> + */
> +static const uint32_t statvar = 1024;
> +static uint32_t statint;	/* number of clock ticks for
stathz */
> +static uint32_t statmin;	/* minimum stat clock count
in ticks */
> +static uint32_t statprev;/* last value of we set
statclock to */
> +static u_int statcountperusec;	/* number of ticks per
usec at current stathz */
> +
> +/*
>   * Common parts of todclock autoconfiguration.
>   */
>  void
>  -63,12 +77,21 
>  cpu_initclocks(void)
>  {
>  
> -	/* start timer */
> +	if (stathz == 0)
> +		stathz = hz;
> +
> +	if (profhz == 0)
> +		profhz = hz * 5;
> +
> +	setstatclockrate(stathz);
> +
> +	/* start timer interrups for hardclock */
>  	if (timer_start == NULL)
>  		panic("cpu_initclocks(): no timer
configured");
>  	(*timer_start)(timer_cookie);
>  
> -	return;
> +	/* enable statclock intr (CPU INT5) */
> +	_splnone();
>  }
>  
>  /*
>  -139,11 +162,59 
>  }
>  
>  void
> -setstatclockrate(int arg)
> +setstatclockrate(int newhz)
>  {
> -	/* XXX */
> +	uint32_t countpersecond, statvarticks;
> +
> +	statprev = mips3_cp0_count_read();
> +
> +	statint = ((curcpu()->ci_cpu_freq + newhz / 2) /
newhz) / 2;
> +
> +	/* Get the total ticks a second */
> +	countpersecond = statint * newhz;
> +
> +	/* now work out how many ticks per usec */
> +	statcountperusec = countpersecond / 1000000;
> +
> +	/* calculate a variance range of statvar */
> +	statvarticks = statcountperusec * statvar;
> +
> +	/* minimum is statint - 50% of variant */
> +	statmin = statint - (statvarticks / 2);
> +
> +	mips3_cp0_compare_write(statprev + statint);
> +}
> +
> +void
> +statclockintr(struct clockframe *cfp)
> +{
> +	uint32_t curcount, statnext, delta, r;
> +	int lost;
> +
> +	lost = 0;
> +
> +	do {
> +		r = (uint32_t)random() & (statvar - 1);
> +	} while (r == 0);
> +	statnext = statprev + statmin + (r *
statcountperusec);
> +
> +	mips3_cp0_compare_write(statnext);
> +	curcount = mips3_cp0_count_read();
> +	delta = statnext - curcount;
> +
> +	while ((int32_t)delta < 0) {
> +		lost++;
> +		delta += statint;
> +	}
> +	if (lost > 0) {
> +		statnext = curcount + delta;
> +		mips3_cp0_compare_write(statnext);
> +		for (; lost > 0; lost--)
> +			statclock(cfp);
> +	}
> +	statclock(cfp);
>  
> -	return;
> +	statprev = statnext;
>  }
>  
>  void
> Index: cobalt/clockvar.h
>
============================================================
=======
> RCS file:
/cvsroot/src/sys/arch/cobalt/cobalt/clockvar.h,v
> retrieving revision 1.3
> diff -u -r1.3 clockvar.h
> --- cobalt/clockvar.h	21 Apr 2006 18:17:45 -0000	1.3
> +++ cobalt/clockvar.h	22 Apr 2006 01:21:48 -0000
>  -25,7 +25,10 
>   * OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
>   */
>  
> +void statclockintr(struct clockframe *);
> +
>  extern struct evcnt hardclock_ev;
> +extern struct evcnt statclock_ev;
>  
>  extern void (*timer_start)(void *);
>  extern long (*timer_read)(void *);
> Index: cobalt/machdep.c
>
============================================================
=======
> RCS file:
/cvsroot/src/sys/arch/cobalt/cobalt/machdep.c,v
> retrieving revision 1.66
> diff -u -r1.66 machdep.c
> --- cobalt/machdep.c	21 Apr 2006 18:21:30 -0000	1.66
> +++ cobalt/machdep.c	22 Apr 2006 01:21:48 -0000
>  -110,6 +110,9 
>  
>  struct evcnt hardclock_ev =
>      EVCNT_INITIALIZER(EVCNT_TYPE_INTR, NULL,
"cpu", "hardclock");
> +struct evcnt statclock_ev =
> +    EVCNT_INITIALIZER(EVCNT_TYPE_INTR, NULL,
"cpu", "statclock");
> +
>  
>  u_int cobalt_id;
>  static const char * const cobalt_model[] =
>  -451,11 +454,22 
>  cpu_intr(uint32_t status, uint32_t cause, uint32_t pc,
uint32_t ipending)
>  {
>  	struct clockframe cf;
> -	static uint32_t cycles;
>  	struct cobalt_intrhand *ih;
>  
>  	uvmexp.intrs++;
>  
> +	if (ipending & MIPS_INT_MASK_5) {
> +		/* statclock */
> +		cf.pc = pc;
> +		cf.sr = status;
> +
> +		statclockintr(&cf);
> +		statclock_ev.ev_count++;
> +
> +		cause &= ~MIPS_INT_MASK_5;
> +	}
> +	_splset((status & MIPS_INT_MASK_5) |
MIPS_SR_INT_IE);
> +
>  	if (ipending & MIPS_INT_MASK_0) {
>  		/* GT64x11 timer0 for hardclock */
>  		volatile uint32_t *irq_src =
>  -467,6 +481,29 
>  			cf.pc = pc;
>  			cf.sr = status;
>  
> +			if ((status & MIPS_INT_MASK) == MIPS_INT_MASK)
{
> +				if ((ipending & MIPS_INT_MASK &
> +				     ~MIPS_INT_MASK_0) == 0) {
> +					/*
> +					 * If all interrupts were enabled and
> +					 * there is no pending interrupts,
> +					 * set MIPS_SR_INT_IE so that
> +					 * spllowerclock() in hardclock()
> +					 * works properly.
> +					 */
> +#if 0					/* MIPS_SR_INT_IE is enabled above */
> +					_splset(MIPS_SR_INT_IE);
> +#endif
> + 				} else {
> +					/*
> +					 * If there are any pending interrputs,
> +					 * clear MIPS_SR_INT_IE in cf.sr so that
> +					 * spllowerclock() in hardclock() will
> +					 * not happen.
> +					 */
> +					cf.sr &= ~MIPS_SR_INT_IE;
> +				}
> +			}
>  			hardclock(&cf);
>  			hardclock_ev.ev_count++;
>  		}
>  -474,20 +511,6 
>  	}
>  	_splset((status & ~cause &
MIPS_HARD_INT_MASK) | MIPS_SR_INT_IE);
>  
> -	if (ipending & MIPS_INT_MASK_5) {
> -		cycles = mips3_cp0_count_read();
> -		mips3_cp0_compare_write(cycles + 1250000);	/* XXX */
> -
> -#if 0
> -		cf.pc = pc;
> -		cf.sr = status;
> -
> -		statclock(&cf);
> -#endif
> -		cause &= ~MIPS_INT_MASK_5;
> -	}
> -	_splset((status & ~cause &
MIPS_HARD_INT_MASK) | MIPS_SR_INT_IE);
> -
>  	if (ipending & MIPS_INT_MASK_3) {
>  		/* 16650 serial */
>  		ih = &intrtab[3];
> Index: conf/std.cobalt
>
============================================================
=======
> RCS file:
/cvsroot/src/sys/arch/cobalt/conf/std.cobalt,v
> retrieving revision 1.12
> diff -u -r1.12 std.cobalt
> --- conf/std.cobalt	11 Dec 2005 12:17:06 -0000	1.12
> +++ conf/std.cobalt	22 Apr 2006 01:21:48 -0000
>  -5,6 +5,7 
>  makeoptions	MACHINE_ARCH="mipsel"
>  
>  options 	MIPS3
> +options 	MIPS3_ENABLE_CLOCK_INTR
>  
>  options 	EXEC_ELF32	# exec ELF32 binaries
>  options 	EXEC_SCRIPT	# exec #! scripts
> Index: include/intr.h
>
============================================================
=======
> RCS file: /cvsroot/src/sys/arch/cobalt/include/intr.h,v
> retrieving revision 1.20
> diff -u -r1.20 intr.h
> --- include/intr.h	21 Apr 2006 18:17:45 -0000	1.20
> +++ include/intr.h	22 Apr 2006 01:21:48 -0000
>  -80,7 +80,8 
>  #define SPLBIO		(SPLSOFT | MIPS_INT_MASK_4)
>  #define SPLNET		(SPLBIO | MIPS_INT_MASK_1 |
MIPS_INT_MASK_2)
>  #define SPLTTY		(SPLNET | MIPS_INT_MASK_3)
> -#define SPLCLOCK	(SPLTTY | MIPS_INT_MASK_0 |
MIPS_INT_MASK_5)
> +#define SPLCLOCK	(SPLTTY | MIPS_INT_MASK_0)
> +#define SPLSTATCLOCK	(SPLCLOCK | MIPS_INT_MASK_5)
>  #define splbio()	_splraise(SPLBIO)
>  #define splnet()	_splraise(SPLNET)
>  #define spltty()	_splraise(SPLTTY)
>  -88,7 +89,7 
>  #define splserial()	_splraise(SPLTTY)
>  #define splclock()	_splraise(SPLCLOCK)
>  #define splvm()		splclock()
> -#define splstatclock()	splclock()
> +#define splstatclock()	_splraise(SPLSTATCLOCK)
>  #define spllowersoftclock()
_spllower(MIPS_SOFT_INT_MASK_0)
>  
>  #define	splsched()	splhigh()
>   


-- 
Garrett D'Amore, Principal Software Engineer
Tadpole Computer / Computing Technologies Division,
General Dynamics C4 Systems
http://www.tadpolecom
puter.com/
Phone: 951 325-2134  Fax: 951 325-2191

In article <4449D2AF.2070308tadpole.com>
garrett_damoretadpole.com wrote:

> Out of curiosity, why not use CPU INT5 (CP0 clock
interrupt) for
> hardclock? 

Not sure, but I guess:
- native OSes (Ultrix, WindowsNT etc.) don't use it
- MIPS1 CPUs don't have it (but have common external
timers?)
- external timers could be more flexible then internal one
- no certain way to get precise CPU clock frequency on all
models
  (but external timer clock is CPU indepenedent?)
etc?

> I never even knew there was support for a separate
statistics clock.

It is mentioned in the D&I of 4.4BSD section 3.4,
but I don't know it's still useful for modern systems.
(the example in the book is hp300)
---
Izumi Tsutsui