1 /* 2 * Copyright (c) 2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/clocksource.h> 34 #include <linux/highmem.h> 35 #include <linux/ptp_clock_kernel.h> 36 #include <rdma/mlx5-abi.h> 37 #include "lib/eq.h" 38 #include "en.h" 39 #include "clock.h" 40 41 enum { 42 MLX5_CYCLES_SHIFT = 23 43 }; 44 45 enum { 46 MLX5_PIN_MODE_IN = 0x0, 47 MLX5_PIN_MODE_OUT = 0x1, 48 }; 49 50 enum { 51 MLX5_OUT_PATTERN_PULSE = 0x0, 52 MLX5_OUT_PATTERN_PERIODIC = 0x1, 53 }; 54 55 enum { 56 MLX5_EVENT_MODE_DISABLE = 0x0, 57 MLX5_EVENT_MODE_REPETETIVE = 0x1, 58 MLX5_EVENT_MODE_ONCE_TILL_ARM = 0x2, 59 }; 60 61 enum { 62 MLX5_MTPPS_FS_ENABLE = BIT(0x0), 63 MLX5_MTPPS_FS_PATTERN = BIT(0x2), 64 MLX5_MTPPS_FS_PIN_MODE = BIT(0x3), 65 MLX5_MTPPS_FS_TIME_STAMP = BIT(0x4), 66 MLX5_MTPPS_FS_OUT_PULSE_DURATION = BIT(0x5), 67 MLX5_MTPPS_FS_ENH_OUT_PER_ADJ = BIT(0x7), 68 }; 69 70 static bool mlx5_real_time_mode(struct mlx5_core_dev *mdev) 71 { 72 return (mlx5_is_real_time_rq(mdev) || mlx5_is_real_time_sq(mdev)); 73 } 74 75 static bool mlx5_modify_mtutc_allowed(struct mlx5_core_dev *mdev) 76 { 77 return MLX5_CAP_MCAM_FEATURE(mdev, ptpcyc2realtime_modify); 78 } 79 80 static int mlx5_set_mtutc(struct mlx5_core_dev *dev, u32 *mtutc, u32 size) 81 { 82 u32 out[MLX5_ST_SZ_DW(mtutc_reg)] = {}; 83 84 if (!MLX5_CAP_MCAM_REG(dev, mtutc)) 85 return -EOPNOTSUPP; 86 87 return mlx5_core_access_reg(dev, mtutc, size, out, sizeof(out), 88 MLX5_REG_MTUTC, 0, 1); 89 } 90 91 static u64 mlx5_read_time(struct mlx5_core_dev *dev, 92 struct ptp_system_timestamp *sts, 93 bool real_time) 94 { 95 u32 timer_h, timer_h1, timer_l; 96 97 timer_h = ioread32be(real_time ? &dev->iseg->real_time_h : 98 &dev->iseg->internal_timer_h); 99 ptp_read_system_prets(sts); 100 timer_l = ioread32be(real_time ? &dev->iseg->real_time_l : 101 &dev->iseg->internal_timer_l); 102 ptp_read_system_postts(sts); 103 timer_h1 = ioread32be(real_time ? &dev->iseg->real_time_h : 104 &dev->iseg->internal_timer_h); 105 if (timer_h != timer_h1) { 106 /* wrap around */ 107 ptp_read_system_prets(sts); 108 timer_l = ioread32be(real_time ? &dev->iseg->real_time_l : 109 &dev->iseg->internal_timer_l); 110 ptp_read_system_postts(sts); 111 } 112 113 return real_time ? REAL_TIME_TO_NS(timer_h1, timer_l) : 114 (u64)timer_l | (u64)timer_h1 << 32; 115 } 116 117 static u64 read_internal_timer(const struct cyclecounter *cc) 118 { 119 struct mlx5_timer *timer = container_of(cc, struct mlx5_timer, cycles); 120 struct mlx5_clock *clock = container_of(timer, struct mlx5_clock, timer); 121 struct mlx5_core_dev *mdev = container_of(clock, struct mlx5_core_dev, 122 clock); 123 124 return mlx5_read_time(mdev, NULL, false) & cc->mask; 125 } 126 127 static void mlx5_update_clock_info_page(struct mlx5_core_dev *mdev) 128 { 129 struct mlx5_ib_clock_info *clock_info = mdev->clock_info; 130 struct mlx5_clock *clock = &mdev->clock; 131 struct mlx5_timer *timer; 132 u32 sign; 133 134 if (!clock_info) 135 return; 136 137 sign = smp_load_acquire(&clock_info->sign); 138 smp_store_mb(clock_info->sign, 139 sign | MLX5_IB_CLOCK_INFO_KERNEL_UPDATING); 140 141 timer = &clock->timer; 142 clock_info->cycles = timer->tc.cycle_last; 143 clock_info->mult = timer->cycles.mult; 144 clock_info->nsec = timer->tc.nsec; 145 clock_info->frac = timer->tc.frac; 146 147 smp_store_release(&clock_info->sign, 148 sign + MLX5_IB_CLOCK_INFO_KERNEL_UPDATING * 2); 149 } 150 151 static void mlx5_pps_out(struct work_struct *work) 152 { 153 struct mlx5_pps *pps_info = container_of(work, struct mlx5_pps, 154 out_work); 155 struct mlx5_clock *clock = container_of(pps_info, struct mlx5_clock, 156 pps_info); 157 struct mlx5_core_dev *mdev = container_of(clock, struct mlx5_core_dev, 158 clock); 159 u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {0}; 160 unsigned long flags; 161 int i; 162 163 for (i = 0; i < clock->ptp_info.n_pins; i++) { 164 u64 tstart; 165 166 write_seqlock_irqsave(&clock->lock, flags); 167 tstart = clock->pps_info.start[i]; 168 clock->pps_info.start[i] = 0; 169 write_sequnlock_irqrestore(&clock->lock, flags); 170 if (!tstart) 171 continue; 172 173 MLX5_SET(mtpps_reg, in, pin, i); 174 MLX5_SET64(mtpps_reg, in, time_stamp, tstart); 175 MLX5_SET(mtpps_reg, in, field_select, MLX5_MTPPS_FS_TIME_STAMP); 176 mlx5_set_mtpps(mdev, in, sizeof(in)); 177 } 178 } 179 180 static void mlx5_timestamp_overflow(struct work_struct *work) 181 { 182 struct delayed_work *dwork = to_delayed_work(work); 183 struct mlx5_core_dev *mdev; 184 struct mlx5_timer *timer; 185 struct mlx5_clock *clock; 186 unsigned long flags; 187 188 timer = container_of(dwork, struct mlx5_timer, overflow_work); 189 clock = container_of(timer, struct mlx5_clock, timer); 190 mdev = container_of(clock, struct mlx5_core_dev, clock); 191 192 write_seqlock_irqsave(&clock->lock, flags); 193 timecounter_read(&timer->tc); 194 mlx5_update_clock_info_page(mdev); 195 write_sequnlock_irqrestore(&clock->lock, flags); 196 schedule_delayed_work(&timer->overflow_work, timer->overflow_period); 197 } 198 199 static int mlx5_ptp_settime_real_time(struct mlx5_core_dev *mdev, 200 const struct timespec64 *ts) 201 { 202 u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {}; 203 204 if (!mlx5_modify_mtutc_allowed(mdev)) 205 return 0; 206 207 if (ts->tv_sec < 0 || ts->tv_sec > U32_MAX || 208 ts->tv_nsec < 0 || ts->tv_nsec > NSEC_PER_SEC) 209 return -EINVAL; 210 211 MLX5_SET(mtutc_reg, in, operation, MLX5_MTUTC_OPERATION_SET_TIME_IMMEDIATE); 212 MLX5_SET(mtutc_reg, in, utc_sec, ts->tv_sec); 213 MLX5_SET(mtutc_reg, in, utc_nsec, ts->tv_nsec); 214 215 return mlx5_set_mtutc(mdev, in, sizeof(in)); 216 } 217 218 static int mlx5_ptp_settime(struct ptp_clock_info *ptp, const struct timespec64 *ts) 219 { 220 struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info); 221 struct mlx5_timer *timer = &clock->timer; 222 struct mlx5_core_dev *mdev; 223 unsigned long flags; 224 int err; 225 226 mdev = container_of(clock, struct mlx5_core_dev, clock); 227 err = mlx5_ptp_settime_real_time(mdev, ts); 228 if (err) 229 return err; 230 231 write_seqlock_irqsave(&clock->lock, flags); 232 timecounter_init(&timer->tc, &timer->cycles, timespec64_to_ns(ts)); 233 mlx5_update_clock_info_page(mdev); 234 write_sequnlock_irqrestore(&clock->lock, flags); 235 236 return 0; 237 } 238 239 static 240 struct timespec64 mlx5_ptp_gettimex_real_time(struct mlx5_core_dev *mdev, 241 struct ptp_system_timestamp *sts) 242 { 243 struct timespec64 ts; 244 u64 time; 245 246 time = mlx5_read_time(mdev, sts, true); 247 ts = ns_to_timespec64(time); 248 return ts; 249 } 250 251 static int mlx5_ptp_gettimex(struct ptp_clock_info *ptp, struct timespec64 *ts, 252 struct ptp_system_timestamp *sts) 253 { 254 struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info); 255 struct mlx5_timer *timer = &clock->timer; 256 struct mlx5_core_dev *mdev; 257 unsigned long flags; 258 u64 cycles, ns; 259 260 mdev = container_of(clock, struct mlx5_core_dev, clock); 261 if (mlx5_real_time_mode(mdev)) { 262 *ts = mlx5_ptp_gettimex_real_time(mdev, sts); 263 goto out; 264 } 265 266 write_seqlock_irqsave(&clock->lock, flags); 267 cycles = mlx5_read_time(mdev, sts, false); 268 ns = timecounter_cyc2time(&timer->tc, cycles); 269 write_sequnlock_irqrestore(&clock->lock, flags); 270 *ts = ns_to_timespec64(ns); 271 out: 272 return 0; 273 } 274 275 static int mlx5_ptp_adjtime_real_time(struct mlx5_core_dev *mdev, s64 delta) 276 { 277 u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {}; 278 279 if (!mlx5_modify_mtutc_allowed(mdev)) 280 return 0; 281 282 /* HW time adjustment range is s16. If out of range, settime instead */ 283 if (delta < S16_MIN || delta > S16_MAX) { 284 struct timespec64 ts; 285 s64 ns; 286 287 ts = mlx5_ptp_gettimex_real_time(mdev, NULL); 288 ns = timespec64_to_ns(&ts) + delta; 289 ts = ns_to_timespec64(ns); 290 return mlx5_ptp_settime_real_time(mdev, &ts); 291 } 292 293 MLX5_SET(mtutc_reg, in, operation, MLX5_MTUTC_OPERATION_ADJUST_TIME); 294 MLX5_SET(mtutc_reg, in, time_adjustment, delta); 295 296 return mlx5_set_mtutc(mdev, in, sizeof(in)); 297 } 298 299 static int mlx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) 300 { 301 struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info); 302 struct mlx5_timer *timer = &clock->timer; 303 struct mlx5_core_dev *mdev; 304 unsigned long flags; 305 int err; 306 307 mdev = container_of(clock, struct mlx5_core_dev, clock); 308 309 err = mlx5_ptp_adjtime_real_time(mdev, delta); 310 if (err) 311 return err; 312 write_seqlock_irqsave(&clock->lock, flags); 313 timecounter_adjtime(&timer->tc, delta); 314 mlx5_update_clock_info_page(mdev); 315 write_sequnlock_irqrestore(&clock->lock, flags); 316 317 return 0; 318 } 319 320 static int mlx5_ptp_adjfreq_real_time(struct mlx5_core_dev *mdev, s32 freq) 321 { 322 u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {}; 323 324 if (!mlx5_modify_mtutc_allowed(mdev)) 325 return 0; 326 327 MLX5_SET(mtutc_reg, in, operation, MLX5_MTUTC_OPERATION_ADJUST_FREQ_UTC); 328 MLX5_SET(mtutc_reg, in, freq_adjustment, freq); 329 330 return mlx5_set_mtutc(mdev, in, sizeof(in)); 331 } 332 333 static int mlx5_ptp_adjfreq(struct ptp_clock_info *ptp, s32 delta) 334 { 335 struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info); 336 struct mlx5_timer *timer = &clock->timer; 337 struct mlx5_core_dev *mdev; 338 unsigned long flags; 339 int neg_adj = 0; 340 u32 diff; 341 u64 adj; 342 int err; 343 344 mdev = container_of(clock, struct mlx5_core_dev, clock); 345 err = mlx5_ptp_adjfreq_real_time(mdev, delta); 346 if (err) 347 return err; 348 349 if (delta < 0) { 350 neg_adj = 1; 351 delta = -delta; 352 } 353 354 adj = timer->nominal_c_mult; 355 adj *= delta; 356 diff = div_u64(adj, 1000000000ULL); 357 358 write_seqlock_irqsave(&clock->lock, flags); 359 timecounter_read(&timer->tc); 360 timer->cycles.mult = neg_adj ? timer->nominal_c_mult - diff : 361 timer->nominal_c_mult + diff; 362 mlx5_update_clock_info_page(mdev); 363 write_sequnlock_irqrestore(&clock->lock, flags); 364 365 return 0; 366 } 367 368 static int mlx5_extts_configure(struct ptp_clock_info *ptp, 369 struct ptp_clock_request *rq, 370 int on) 371 { 372 struct mlx5_clock *clock = 373 container_of(ptp, struct mlx5_clock, ptp_info); 374 struct mlx5_core_dev *mdev = 375 container_of(clock, struct mlx5_core_dev, clock); 376 u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {0}; 377 u32 field_select = 0; 378 u8 pin_mode = 0; 379 u8 pattern = 0; 380 int pin = -1; 381 int err = 0; 382 383 if (!MLX5_PPS_CAP(mdev)) 384 return -EOPNOTSUPP; 385 386 /* Reject requests with unsupported flags */ 387 if (rq->extts.flags & ~(PTP_ENABLE_FEATURE | 388 PTP_RISING_EDGE | 389 PTP_FALLING_EDGE | 390 PTP_STRICT_FLAGS)) 391 return -EOPNOTSUPP; 392 393 /* Reject requests to enable time stamping on both edges. */ 394 if ((rq->extts.flags & PTP_STRICT_FLAGS) && 395 (rq->extts.flags & PTP_ENABLE_FEATURE) && 396 (rq->extts.flags & PTP_EXTTS_EDGES) == PTP_EXTTS_EDGES) 397 return -EOPNOTSUPP; 398 399 if (rq->extts.index >= clock->ptp_info.n_pins) 400 return -EINVAL; 401 402 pin = ptp_find_pin(clock->ptp, PTP_PF_EXTTS, rq->extts.index); 403 if (pin < 0) 404 return -EBUSY; 405 406 if (on) { 407 pin_mode = MLX5_PIN_MODE_IN; 408 pattern = !!(rq->extts.flags & PTP_FALLING_EDGE); 409 field_select = MLX5_MTPPS_FS_PIN_MODE | 410 MLX5_MTPPS_FS_PATTERN | 411 MLX5_MTPPS_FS_ENABLE; 412 } else { 413 field_select = MLX5_MTPPS_FS_ENABLE; 414 } 415 416 MLX5_SET(mtpps_reg, in, pin, pin); 417 MLX5_SET(mtpps_reg, in, pin_mode, pin_mode); 418 MLX5_SET(mtpps_reg, in, pattern, pattern); 419 MLX5_SET(mtpps_reg, in, enable, on); 420 MLX5_SET(mtpps_reg, in, field_select, field_select); 421 422 err = mlx5_set_mtpps(mdev, in, sizeof(in)); 423 if (err) 424 return err; 425 426 return mlx5_set_mtppse(mdev, pin, 0, 427 MLX5_EVENT_MODE_REPETETIVE & on); 428 } 429 430 static u64 find_target_cycles(struct mlx5_core_dev *mdev, s64 target_ns) 431 { 432 struct mlx5_clock *clock = &mdev->clock; 433 u64 cycles_now, cycles_delta; 434 u64 nsec_now, nsec_delta; 435 struct mlx5_timer *timer; 436 unsigned long flags; 437 438 timer = &clock->timer; 439 440 cycles_now = mlx5_read_time(mdev, NULL, false); 441 write_seqlock_irqsave(&clock->lock, flags); 442 nsec_now = timecounter_cyc2time(&timer->tc, cycles_now); 443 nsec_delta = target_ns - nsec_now; 444 cycles_delta = div64_u64(nsec_delta << timer->cycles.shift, 445 timer->cycles.mult); 446 write_sequnlock_irqrestore(&clock->lock, flags); 447 448 return cycles_now + cycles_delta; 449 } 450 451 static u64 perout_conf_internal_timer(struct mlx5_core_dev *mdev, s64 sec) 452 { 453 struct timespec64 ts = {}; 454 s64 target_ns; 455 456 ts.tv_sec = sec; 457 target_ns = timespec64_to_ns(&ts); 458 459 return find_target_cycles(mdev, target_ns); 460 } 461 462 static u64 perout_conf_real_time(s64 sec) 463 { 464 return (u64)sec << 32; 465 } 466 467 static int mlx5_perout_configure(struct ptp_clock_info *ptp, 468 struct ptp_clock_request *rq, 469 int on) 470 { 471 struct mlx5_clock *clock = 472 container_of(ptp, struct mlx5_clock, ptp_info); 473 struct mlx5_core_dev *mdev = 474 container_of(clock, struct mlx5_core_dev, clock); 475 bool rt_mode = mlx5_real_time_mode(mdev); 476 u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {0}; 477 struct timespec64 ts; 478 u32 field_select = 0; 479 u64 time_stamp = 0; 480 u8 pin_mode = 0; 481 u8 pattern = 0; 482 int pin = -1; 483 int err = 0; 484 s64 ns; 485 486 if (!MLX5_PPS_CAP(mdev)) 487 return -EOPNOTSUPP; 488 489 /* Reject requests with unsupported flags */ 490 if (rq->perout.flags) 491 return -EOPNOTSUPP; 492 493 if (rq->perout.index >= clock->ptp_info.n_pins) 494 return -EINVAL; 495 496 field_select = MLX5_MTPPS_FS_ENABLE; 497 pin = ptp_find_pin(clock->ptp, PTP_PF_PEROUT, rq->perout.index); 498 if (pin < 0) 499 return -EBUSY; 500 501 if (on) { 502 bool rt_mode = mlx5_real_time_mode(mdev); 503 s64 sec = rq->perout.start.sec; 504 505 if (rq->perout.start.nsec) 506 return -EINVAL; 507 508 pin_mode = MLX5_PIN_MODE_OUT; 509 pattern = MLX5_OUT_PATTERN_PERIODIC; 510 ts.tv_sec = rq->perout.period.sec; 511 ts.tv_nsec = rq->perout.period.nsec; 512 ns = timespec64_to_ns(&ts); 513 514 if ((ns >> 1) != 500000000LL) 515 return -EINVAL; 516 517 if (rt_mode && sec > U32_MAX) 518 return -EINVAL; 519 520 time_stamp = rt_mode ? perout_conf_real_time(sec) : 521 perout_conf_internal_timer(mdev, sec); 522 523 field_select |= MLX5_MTPPS_FS_PIN_MODE | 524 MLX5_MTPPS_FS_PATTERN | 525 MLX5_MTPPS_FS_TIME_STAMP; 526 } 527 528 MLX5_SET(mtpps_reg, in, pin, pin); 529 MLX5_SET(mtpps_reg, in, pin_mode, pin_mode); 530 MLX5_SET(mtpps_reg, in, pattern, pattern); 531 MLX5_SET(mtpps_reg, in, enable, on); 532 MLX5_SET64(mtpps_reg, in, time_stamp, time_stamp); 533 MLX5_SET(mtpps_reg, in, field_select, field_select); 534 535 err = mlx5_set_mtpps(mdev, in, sizeof(in)); 536 if (err) 537 return err; 538 539 if (rt_mode) 540 return 0; 541 542 return mlx5_set_mtppse(mdev, pin, 0, 543 MLX5_EVENT_MODE_REPETETIVE & on); 544 } 545 546 static int mlx5_pps_configure(struct ptp_clock_info *ptp, 547 struct ptp_clock_request *rq, 548 int on) 549 { 550 struct mlx5_clock *clock = 551 container_of(ptp, struct mlx5_clock, ptp_info); 552 553 clock->pps_info.enabled = !!on; 554 return 0; 555 } 556 557 static int mlx5_ptp_enable(struct ptp_clock_info *ptp, 558 struct ptp_clock_request *rq, 559 int on) 560 { 561 switch (rq->type) { 562 case PTP_CLK_REQ_EXTTS: 563 return mlx5_extts_configure(ptp, rq, on); 564 case PTP_CLK_REQ_PEROUT: 565 return mlx5_perout_configure(ptp, rq, on); 566 case PTP_CLK_REQ_PPS: 567 return mlx5_pps_configure(ptp, rq, on); 568 default: 569 return -EOPNOTSUPP; 570 } 571 return 0; 572 } 573 574 enum { 575 MLX5_MTPPS_REG_CAP_PIN_X_MODE_SUPPORT_PPS_IN = BIT(0), 576 MLX5_MTPPS_REG_CAP_PIN_X_MODE_SUPPORT_PPS_OUT = BIT(1), 577 }; 578 579 static int mlx5_ptp_verify(struct ptp_clock_info *ptp, unsigned int pin, 580 enum ptp_pin_function func, unsigned int chan) 581 { 582 struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, 583 ptp_info); 584 585 switch (func) { 586 case PTP_PF_NONE: 587 return 0; 588 case PTP_PF_EXTTS: 589 return !(clock->pps_info.pin_caps[pin] & 590 MLX5_MTPPS_REG_CAP_PIN_X_MODE_SUPPORT_PPS_IN); 591 case PTP_PF_PEROUT: 592 return !(clock->pps_info.pin_caps[pin] & 593 MLX5_MTPPS_REG_CAP_PIN_X_MODE_SUPPORT_PPS_OUT); 594 default: 595 return -EOPNOTSUPP; 596 } 597 } 598 599 static const struct ptp_clock_info mlx5_ptp_clock_info = { 600 .owner = THIS_MODULE, 601 .name = "mlx5_ptp", 602 .max_adj = 100000000, 603 .n_alarm = 0, 604 .n_ext_ts = 0, 605 .n_per_out = 0, 606 .n_pins = 0, 607 .pps = 0, 608 .adjfreq = mlx5_ptp_adjfreq, 609 .adjtime = mlx5_ptp_adjtime, 610 .gettimex64 = mlx5_ptp_gettimex, 611 .settime64 = mlx5_ptp_settime, 612 .enable = NULL, 613 .verify = NULL, 614 }; 615 616 static int mlx5_query_mtpps_pin_mode(struct mlx5_core_dev *mdev, u8 pin, 617 u32 *mtpps, u32 mtpps_size) 618 { 619 u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {}; 620 621 MLX5_SET(mtpps_reg, in, pin, pin); 622 623 return mlx5_core_access_reg(mdev, in, sizeof(in), mtpps, 624 mtpps_size, MLX5_REG_MTPPS, 0, 0); 625 } 626 627 static int mlx5_get_pps_pin_mode(struct mlx5_clock *clock, u8 pin) 628 { 629 struct mlx5_core_dev *mdev = container_of(clock, struct mlx5_core_dev, clock); 630 631 u32 out[MLX5_ST_SZ_DW(mtpps_reg)] = {}; 632 u8 mode; 633 int err; 634 635 err = mlx5_query_mtpps_pin_mode(mdev, pin, out, sizeof(out)); 636 if (err || !MLX5_GET(mtpps_reg, out, enable)) 637 return PTP_PF_NONE; 638 639 mode = MLX5_GET(mtpps_reg, out, pin_mode); 640 641 if (mode == MLX5_PIN_MODE_IN) 642 return PTP_PF_EXTTS; 643 else if (mode == MLX5_PIN_MODE_OUT) 644 return PTP_PF_PEROUT; 645 646 return PTP_PF_NONE; 647 } 648 649 static void mlx5_init_pin_config(struct mlx5_clock *clock) 650 { 651 int i; 652 653 if (!clock->ptp_info.n_pins) 654 return; 655 656 clock->ptp_info.pin_config = 657 kcalloc(clock->ptp_info.n_pins, 658 sizeof(*clock->ptp_info.pin_config), 659 GFP_KERNEL); 660 if (!clock->ptp_info.pin_config) 661 return; 662 clock->ptp_info.enable = mlx5_ptp_enable; 663 clock->ptp_info.verify = mlx5_ptp_verify; 664 clock->ptp_info.pps = 1; 665 666 for (i = 0; i < clock->ptp_info.n_pins; i++) { 667 snprintf(clock->ptp_info.pin_config[i].name, 668 sizeof(clock->ptp_info.pin_config[i].name), 669 "mlx5_pps%d", i); 670 clock->ptp_info.pin_config[i].index = i; 671 clock->ptp_info.pin_config[i].func = mlx5_get_pps_pin_mode(clock, i); 672 clock->ptp_info.pin_config[i].chan = 0; 673 } 674 } 675 676 static void mlx5_get_pps_caps(struct mlx5_core_dev *mdev) 677 { 678 struct mlx5_clock *clock = &mdev->clock; 679 u32 out[MLX5_ST_SZ_DW(mtpps_reg)] = {0}; 680 681 mlx5_query_mtpps(mdev, out, sizeof(out)); 682 683 clock->ptp_info.n_pins = MLX5_GET(mtpps_reg, out, 684 cap_number_of_pps_pins); 685 clock->ptp_info.n_ext_ts = MLX5_GET(mtpps_reg, out, 686 cap_max_num_of_pps_in_pins); 687 clock->ptp_info.n_per_out = MLX5_GET(mtpps_reg, out, 688 cap_max_num_of_pps_out_pins); 689 690 clock->pps_info.pin_caps[0] = MLX5_GET(mtpps_reg, out, cap_pin_0_mode); 691 clock->pps_info.pin_caps[1] = MLX5_GET(mtpps_reg, out, cap_pin_1_mode); 692 clock->pps_info.pin_caps[2] = MLX5_GET(mtpps_reg, out, cap_pin_2_mode); 693 clock->pps_info.pin_caps[3] = MLX5_GET(mtpps_reg, out, cap_pin_3_mode); 694 clock->pps_info.pin_caps[4] = MLX5_GET(mtpps_reg, out, cap_pin_4_mode); 695 clock->pps_info.pin_caps[5] = MLX5_GET(mtpps_reg, out, cap_pin_5_mode); 696 clock->pps_info.pin_caps[6] = MLX5_GET(mtpps_reg, out, cap_pin_6_mode); 697 clock->pps_info.pin_caps[7] = MLX5_GET(mtpps_reg, out, cap_pin_7_mode); 698 } 699 700 static void ts_next_sec(struct timespec64 *ts) 701 { 702 ts->tv_sec += 1; 703 ts->tv_nsec = 0; 704 } 705 706 static u64 perout_conf_next_event_timer(struct mlx5_core_dev *mdev, 707 struct mlx5_clock *clock) 708 { 709 struct timespec64 ts; 710 s64 target_ns; 711 712 mlx5_ptp_gettimex(&clock->ptp_info, &ts, NULL); 713 ts_next_sec(&ts); 714 target_ns = timespec64_to_ns(&ts); 715 716 return find_target_cycles(mdev, target_ns); 717 } 718 719 static int mlx5_pps_event(struct notifier_block *nb, 720 unsigned long type, void *data) 721 { 722 struct mlx5_clock *clock = mlx5_nb_cof(nb, struct mlx5_clock, pps_nb); 723 struct ptp_clock_event ptp_event; 724 struct mlx5_eqe *eqe = data; 725 int pin = eqe->data.pps.pin; 726 struct mlx5_core_dev *mdev; 727 unsigned long flags; 728 u64 ns; 729 730 mdev = container_of(clock, struct mlx5_core_dev, clock); 731 732 switch (clock->ptp_info.pin_config[pin].func) { 733 case PTP_PF_EXTTS: 734 ptp_event.index = pin; 735 ptp_event.timestamp = mlx5_real_time_mode(mdev) ? 736 mlx5_real_time_cyc2time(clock, 737 be64_to_cpu(eqe->data.pps.time_stamp)) : 738 mlx5_timecounter_cyc2time(clock, 739 be64_to_cpu(eqe->data.pps.time_stamp)); 740 if (clock->pps_info.enabled) { 741 ptp_event.type = PTP_CLOCK_PPSUSR; 742 ptp_event.pps_times.ts_real = 743 ns_to_timespec64(ptp_event.timestamp); 744 } else { 745 ptp_event.type = PTP_CLOCK_EXTTS; 746 } 747 /* TODOL clock->ptp can be NULL if ptp_clock_register fails */ 748 ptp_clock_event(clock->ptp, &ptp_event); 749 break; 750 case PTP_PF_PEROUT: 751 ns = perout_conf_next_event_timer(mdev, clock); 752 write_seqlock_irqsave(&clock->lock, flags); 753 clock->pps_info.start[pin] = ns; 754 write_sequnlock_irqrestore(&clock->lock, flags); 755 schedule_work(&clock->pps_info.out_work); 756 break; 757 default: 758 mlx5_core_err(mdev, " Unhandled clock PPS event, func %d\n", 759 clock->ptp_info.pin_config[pin].func); 760 } 761 762 return NOTIFY_OK; 763 } 764 765 static void mlx5_timecounter_init(struct mlx5_core_dev *mdev) 766 { 767 struct mlx5_clock *clock = &mdev->clock; 768 struct mlx5_timer *timer = &clock->timer; 769 u32 dev_freq; 770 771 dev_freq = MLX5_CAP_GEN(mdev, device_frequency_khz); 772 timer->cycles.read = read_internal_timer; 773 timer->cycles.shift = MLX5_CYCLES_SHIFT; 774 timer->cycles.mult = clocksource_khz2mult(dev_freq, 775 timer->cycles.shift); 776 timer->nominal_c_mult = timer->cycles.mult; 777 timer->cycles.mask = CLOCKSOURCE_MASK(41); 778 779 timecounter_init(&timer->tc, &timer->cycles, 780 ktime_to_ns(ktime_get_real())); 781 } 782 783 static void mlx5_init_overflow_period(struct mlx5_clock *clock) 784 { 785 struct mlx5_core_dev *mdev = container_of(clock, struct mlx5_core_dev, clock); 786 struct mlx5_ib_clock_info *clock_info = mdev->clock_info; 787 struct mlx5_timer *timer = &clock->timer; 788 u64 overflow_cycles; 789 u64 frac = 0; 790 u64 ns; 791 792 /* Calculate period in seconds to call the overflow watchdog - to make 793 * sure counter is checked at least twice every wrap around. 794 * The period is calculated as the minimum between max HW cycles count 795 * (The clock source mask) and max amount of cycles that can be 796 * multiplied by clock multiplier where the result doesn't exceed 797 * 64bits. 798 */ 799 overflow_cycles = div64_u64(~0ULL >> 1, timer->cycles.mult); 800 overflow_cycles = min(overflow_cycles, div_u64(timer->cycles.mask, 3)); 801 802 ns = cyclecounter_cyc2ns(&timer->cycles, overflow_cycles, 803 frac, &frac); 804 do_div(ns, NSEC_PER_SEC / HZ); 805 timer->overflow_period = ns; 806 807 INIT_DELAYED_WORK(&timer->overflow_work, mlx5_timestamp_overflow); 808 if (timer->overflow_period) 809 schedule_delayed_work(&timer->overflow_work, 0); 810 else 811 mlx5_core_warn(mdev, 812 "invalid overflow period, overflow_work is not scheduled\n"); 813 814 if (clock_info) 815 clock_info->overflow_period = timer->overflow_period; 816 } 817 818 static void mlx5_init_clock_info(struct mlx5_core_dev *mdev) 819 { 820 struct mlx5_clock *clock = &mdev->clock; 821 struct mlx5_ib_clock_info *info; 822 struct mlx5_timer *timer; 823 824 mdev->clock_info = (struct mlx5_ib_clock_info *)get_zeroed_page(GFP_KERNEL); 825 if (!mdev->clock_info) { 826 mlx5_core_warn(mdev, "Failed to allocate IB clock info page\n"); 827 return; 828 } 829 830 info = mdev->clock_info; 831 timer = &clock->timer; 832 833 info->nsec = timer->tc.nsec; 834 info->cycles = timer->tc.cycle_last; 835 info->mask = timer->cycles.mask; 836 info->mult = timer->nominal_c_mult; 837 info->shift = timer->cycles.shift; 838 info->frac = timer->tc.frac; 839 } 840 841 static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev) 842 { 843 struct mlx5_clock *clock = &mdev->clock; 844 845 mlx5_timecounter_init(mdev); 846 mlx5_init_clock_info(mdev); 847 mlx5_init_overflow_period(clock); 848 clock->ptp_info = mlx5_ptp_clock_info; 849 850 if (mlx5_real_time_mode(mdev)) { 851 struct timespec64 ts; 852 853 ktime_get_real_ts64(&ts); 854 mlx5_ptp_settime(&clock->ptp_info, &ts); 855 } 856 } 857 858 static void mlx5_init_pps(struct mlx5_core_dev *mdev) 859 { 860 struct mlx5_clock *clock = &mdev->clock; 861 862 if (!MLX5_PPS_CAP(mdev)) 863 return; 864 865 mlx5_get_pps_caps(mdev); 866 mlx5_init_pin_config(clock); 867 } 868 869 void mlx5_init_clock(struct mlx5_core_dev *mdev) 870 { 871 struct mlx5_clock *clock = &mdev->clock; 872 873 if (!MLX5_CAP_GEN(mdev, device_frequency_khz)) { 874 mlx5_core_warn(mdev, "invalid device_frequency_khz, aborting HW clock init\n"); 875 return; 876 } 877 878 seqlock_init(&clock->lock); 879 mlx5_init_timer_clock(mdev); 880 INIT_WORK(&clock->pps_info.out_work, mlx5_pps_out); 881 882 /* Configure the PHC */ 883 clock->ptp_info = mlx5_ptp_clock_info; 884 885 /* Initialize 1PPS data structures */ 886 mlx5_init_pps(mdev); 887 888 clock->ptp = ptp_clock_register(&clock->ptp_info, 889 &mdev->pdev->dev); 890 if (IS_ERR(clock->ptp)) { 891 mlx5_core_warn(mdev, "ptp_clock_register failed %ld\n", 892 PTR_ERR(clock->ptp)); 893 clock->ptp = NULL; 894 } 895 896 MLX5_NB_INIT(&clock->pps_nb, mlx5_pps_event, PPS_EVENT); 897 mlx5_eq_notifier_register(mdev, &clock->pps_nb); 898 } 899 900 void mlx5_cleanup_clock(struct mlx5_core_dev *mdev) 901 { 902 struct mlx5_clock *clock = &mdev->clock; 903 904 if (!MLX5_CAP_GEN(mdev, device_frequency_khz)) 905 return; 906 907 mlx5_eq_notifier_unregister(mdev, &clock->pps_nb); 908 if (clock->ptp) { 909 ptp_clock_unregister(clock->ptp); 910 clock->ptp = NULL; 911 } 912 913 cancel_work_sync(&clock->pps_info.out_work); 914 cancel_delayed_work_sync(&clock->timer.overflow_work); 915 916 if (mdev->clock_info) { 917 free_page((unsigned long)mdev->clock_info); 918 mdev->clock_info = NULL; 919 } 920 921 kfree(clock->ptp_info.pin_config); 922 } 923