s390/time: correct use of store clock fast

The result of the store-clock-fast (STCKF) instruction is a bit fuzzy.
It can happen that the value stored on one CPU is smaller than the value
stored on another CPU, although the order of the stores is the other
way around. This can cause deltas of get_tod_clock() values to become
negative when they should not be.

We need to be more careful with store-clock-fast, this patch partially
reverts git commit e4b7b4238e666682555461fa52eecd74652f36bb "time:
always use stckf instead of stck if available". The get_tod_clock()
function now uses the store-clock-extended (STCKE) instruction.
get_tod_clock_fast() can be used if the fuzziness of store-clock-fast
is acceptable e.g. for wait loops local to a CPU.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h
index 8ad8af915032a32573e06548e90d89ea7563349c..819b94d2272054d318fdd12c31b8718a9e4bb0bd 100644 (file)
--- a/arch/s390/include/asm/timex.h
+++ b/arch/s390/include/asm/timex.h
@@ -71,30 +71,30 @@ static inline void local_tick_enable(unsigned long long comp)
 
 typedef unsigned long long cycles_t;
 
-static inline unsigned long long get_tod_clock(void)
-{
-       unsigned long long clk;
-
-#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
-       asm volatile(".insn s,0xb27c0000,%0" : "=Q" (clk) : : "cc");
-#else
-       asm volatile("stck %0" : "=Q" (clk) : : "cc");
-#endif
-       return clk;
-}
-
 static inline void get_tod_clock_ext(char *clk)
 {
        asm volatile("stcke %0" : "=Q" (*clk) : : "cc");
 }
 
-static inline unsigned long long get_tod_clock_xt(void)
+static inline unsigned long long get_tod_clock(void)
 {
        unsigned char clk[16];
        get_tod_clock_ext(clk);
        return *((unsigned long long *)&clk[1]);
 }
 
+static inline unsigned long long get_tod_clock_fast(void)
+{
+#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
+       unsigned long long clk;
+
+       asm volatile("stckf %0" : "=Q" (clk) : : "cc");
+       return clk;
+#else
+       return get_tod_clock();
+#endif
+}
+
 static inline cycles_t get_cycles(void)
 {
        return (cycles_t) get_tod_clock() >> 2;
@@ -125,7 +125,7 @@ extern u64 sched_clock_base_cc;
  */
 static inline unsigned long long get_tod_clock_monotonic(void)
 {
-       return get_tod_clock_xt() - sched_clock_base_cc;
+       return get_tod_clock() - sched_clock_base_cc;
 }
 
 /**

diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c
index f1279dc2e1bcecae2a12ea246096f1458b2de77f..17d62fe5d7b70554c36b8f85b34da23d86c90407 100644 (file)
--- a/arch/s390/kernel/debug.c
+++ b/arch/s390/kernel/debug.c
@@ -867,7 +867,7 @@ static inline void
 debug_finish_entry(debug_info_t * id, debug_entry_t* active, int level,
                        int exception)
 {
-       active->id.stck = get_tod_clock();
+       active->id.stck = get_tod_clock_fast();
        active->id.fields.cpuid = smp_processor_id();
        active->caller = __builtin_return_address(0);
        active->id.fields.exception = exception;

diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index 7f35cb33e5102008244b4fd1376954400f2d0009..7f1f7ac5cf7f8a2c3f3966d4fe96fa23af90ea04 100644 (file)
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -385,7 +385,7 @@ static int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu)
        }
 
        if ((!rc) && (vcpu->arch.sie_block->ckc <
-               get_tod_clock() + vcpu->arch.sie_block->epoch)) {
+               get_tod_clock_fast() + vcpu->arch.sie_block->epoch)) {
                if ((!psw_extint_disabled(vcpu)) &&
                        (vcpu->arch.sie_block->gcr[0] & 0x800ul))
                        rc = 1;
@@ -425,7 +425,7 @@ int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
                goto no_timer;
        }
 
-       now = get_tod_clock() + vcpu->arch.sie_block->epoch;
+       now = get_tod_clock_fast() + vcpu->arch.sie_block->epoch;
        if (vcpu->arch.sie_block->ckc < now) {
                __unset_cpu_idle(vcpu);
                return 0;
@@ -515,7 +515,7 @@ void kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
        }
 
        if ((vcpu->arch.sie_block->ckc <
-               get_tod_clock() + vcpu->arch.sie_block->epoch))
+               get_tod_clock_fast() + vcpu->arch.sie_block->epoch))
                __try_deliver_ckc_interrupt(vcpu);
 
        if (atomic_read(&fi->active)) {

diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c
index 57c87d7d7ede01add784c291f31a82f502ca877e..a9f3d0042d58ba849b8c624f23915d88d03d6f70 100644 (file)
--- a/arch/s390/lib/delay.c
+++ b/arch/s390/lib/delay.c
@@ -44,7 +44,7 @@ static void __udelay_disabled(unsigned long long usecs)
        do {
                set_clock_comparator(end);
                vtime_stop_cpu();
-       } while (get_tod_clock() < end);
+       } while (get_tod_clock_fast() < end);
        lockdep_on();
        __ctl_load(cr0, 0, 0);
        __ctl_load(cr6, 6, 6);
@@ -55,7 +55,7 @@ static void __udelay_enabled(unsigned long long usecs)
 {
        u64 clock_saved, end;
 
-       end = get_tod_clock() + (usecs << 12);
+       end = get_tod_clock_fast() + (usecs << 12);
        do {
                clock_saved = 0;
                if (end < S390_lowcore.clock_comparator) {
@@ -65,7 +65,7 @@ static void __udelay_enabled(unsigned long long usecs)
                vtime_stop_cpu();
                if (clock_saved)
                        local_tick_enable(clock_saved);
-       } while (get_tod_clock() < end);
+       } while (get_tod_clock_fast() < end);
 }
 
 /*
@@ -109,8 +109,8 @@ void udelay_simple(unsigned long long usecs)
 {
        u64 end;
 
-       end = get_tod_clock() + (usecs << 12);
-       while (get_tod_clock() < end)
+       end = get_tod_clock_fast() + (usecs << 12);
+       while (get_tod_clock_fast() < end)
                cpu_relax();
 }
 
@@ -120,10 +120,10 @@ void __ndelay(unsigned long long nsecs)
 
        nsecs <<= 9;
        do_div(nsecs, 125);
-       end = get_tod_clock() + nsecs;
+       end = get_tod_clock_fast() + nsecs;
        if (nsecs & ~0xfffUL)
                __udelay(nsecs >> 12);
-       while (get_tod_clock() < end)
+       while (get_tod_clock_fast() < end)
                barrier();
 }
 EXPORT_SYMBOL(__ndelay);

diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c
index a3aa374799dcf99bd334b6e49c8d7753838fbdc3..1fe264379e0d10b57ddca456ce16b72c23535ccb 100644 (file)
--- a/drivers/s390/char/sclp.c
+++ b/drivers/s390/char/sclp.c
@@ -486,7 +486,7 @@ sclp_sync_wait(void)
        timeout = 0;
        if (timer_pending(&sclp_request_timer)) {
                /* Get timeout TOD value */
-               timeout = get_tod_clock() +
+               timeout = get_tod_clock_fast() +
                          sclp_tod_from_jiffies(sclp_request_timer.expires -
                                                jiffies);
        }
@@ -508,7 +508,7 @@ sclp_sync_wait(void)
        while (sclp_running_state != sclp_running_state_idle) {
                /* Check for expired request timer */
                if (timer_pending(&sclp_request_timer) &&
-                   get_tod_clock() > timeout &&
+                   get_tod_clock_fast() > timeout &&
                    del_timer(&sclp_request_timer))
                        sclp_request_timer.function(sclp_request_timer.data);
                cpu_relax();

diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c
index d7da67a31c77f606ef68445f9da446b521a4abf1..88e35d85d205f7c21de1f81860865386a7845289 100644 (file)
--- a/drivers/s390/cio/cio.c
+++ b/drivers/s390/cio/cio.c
@@ -878,9 +878,9 @@ static void css_reset(void)
                        atomic_inc(&chpid_reset_count);
        }
        /* Wait for machine check for all channel paths. */
-       timeout = get_tod_clock() + (RCHP_TIMEOUT << 12);
+       timeout = get_tod_clock_fast() + (RCHP_TIMEOUT << 12);
        while (atomic_read(&chpid_reset_count) != 0) {
-               if (get_tod_clock() > timeout)
+               if (get_tod_clock_fast() > timeout)
                        break;
                cpu_relax();
        }

diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c
index 8ed52aa4912269405158e300f1c983577b29caec..bbd3e511c771addebda2639675d85664b3191494 100644 (file)
--- a/drivers/s390/cio/qdio_main.c
+++ b/drivers/s390/cio/qdio_main.c
@@ -338,10 +338,10 @@ again:
                retries++;
 
                if (!start_time) {
-                       start_time = get_tod_clock();
+                       start_time = get_tod_clock_fast();
                        goto again;
                }
-               if ((get_tod_clock() - start_time) < QDIO_BUSY_BIT_PATIENCE)
+               if (get_tod_clock_fast() - start_time < QDIO_BUSY_BIT_PATIENCE)
                        goto again;
        }
        if (retries) {
@@ -504,7 +504,7 @@ static int get_inbound_buffer_frontier(struct qdio_q *q)
        int count, stop;
        unsigned char state = 0;
 
-       q->timestamp = get_tod_clock();
+       q->timestamp = get_tod_clock_fast();
 
        /*
         * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
@@ -595,7 +595,7 @@ static inline int qdio_inbound_q_done(struct qdio_q *q)
         * At this point we know, that inbound first_to_check
         * has (probably) not moved (see qdio_inbound_processing).
         */
-       if (get_tod_clock() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) {
+       if (get_tod_clock_fast() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) {
                DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x",
                              q->first_to_check);
                return 1;
@@ -728,7 +728,7 @@ static int get_outbound_buffer_frontier(struct qdio_q *q)
        int count, stop;
        unsigned char state = 0;
 
-       q->timestamp = get_tod_clock();
+       q->timestamp = get_tod_clock_fast();
 
        if (need_siga_sync(q))
                if (((queue_type(q) != QDIO_IQDIO_QFMT) &&