Skip to content

Commit

Permalink
add cl-quil-benchmarking system quilc-perf for perf. benchmarking
Browse files Browse the repository at this point in the history 
This is part of issue "Investigate performance at 72-150 qubits #565".

new modules:

  benchmarking/quilc-perf.lisp - primarily exports benchmark-nq
    function to get timings as nQ increases for various chip and
    program types. This also introduces a "chip preparation" step
    before running any benchmarks. This removes any overhead from
    running the first compile on a chip out of the first benchmark
    run. Currently, there is significant but not tremendous amount of
    initial overhead, but a PR under consideration might add
    significant additional preparation overhead, and these should not
    be be charged to the first run.

  benchmarking/quilc-mon-prof.lisp - builds on quilc-perf.lisp and is
    for monitoring and profiling, as opposed to simple benchmarks, and
    is a WIP of sorts in that, while it's been useful to run for
    probing and experimenting in a REPL, we ;; so far lack really good
    theory of operation and associated modes of ;; running. We hope
    with time to get there. Also, note that some of this is SBCL-only.

Makefile - add targets to run benchmark-nq a couple of ways: `make
  benchmark-nq` calls benchmark-nq one time, and `make
  benchmark-nq-2x` calls it twice. The idea is to capture the
  sometimes significant jump in performance observed between a first
  and second run within the same Lisp environment, due to the effects
  of "warming", caching, and the like.

cl-quil-benchmarking.asd - add new module quilc-perf and depend on 3
  additional systems: metering, qvm-app, sb-sprof
  • Loading branch information
@stylewarning
David, Mark H authored and stylewarning committed Oct 21, 2021
1 parent b9a0eb0 commit 93b3ba3
Show file tree
Hide file tree
Showing 5 changed files with 539 additions and 24 deletions.
13 changes: 12 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,13 +147,24 @@ test-ccl:
###############################################################################
# BENCHMARKS
###############################################################################
.PHONY: benchmark-qasm
.PHONY: benchmark-qasm benchmark-nq benchmark-nq-2x

benchmark-qasm:
$(QUICKLISP) \
--eval "(ql:quickload :cl-quil-benchmarking)" \
--eval "(cl-quil-benchmarking::benchmark-qasm-suite)"

benchmark-nq:
$(QUICKLISP) \
--eval "(ql:quickload :cl-quil-benchmarking)" \
--eval "(cl-quil-benchmarking::benchmark-nq)"

benchmark-nq-2x:
$(QUICKLISP) \
--eval "(ql:quickload :cl-quil-benchmarking)" \
--eval "(cl-quil-benchmarking::benchmark-nq)" \
--eval "(cl-quil-benchmarking::benchmark-nq)"

###############################################################################
# CLEAN
###############################################################################
Expand Down
52 changes: 29 additions & 23 deletions benchmarking/qasm-benchmarks.lisp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@
:do (quil::append-instruction-to-lschedule lschedule instr)
:finally (return (quil::lscheduler-calculate-depth lschedule)))))

(defun benchmark-qasm-suite (&key (timeout 30))
(defun benchmark-qasm-suite (&key (timeout 30) named)
"Run benchmarks from qasm suite. If NAMED is not nil, the specified test(s) will be the ONLY one(s) run; otherwise, all the tests are run. NAMED should be a short name (as shown in the output) of a test, either as a symbol or string, or a list thereof (i.e., matching mutiple tests), to be compared using string-equal. TIMEOUT specifies a timeout in seconds, defaulting to 30 seconds."
(let ((timed-out nil))
(flet ((print-rule ()
(format t "+------------------+----------+-------+----------+~%")))
Expand All @@ -73,28 +74,33 @@
(quil::*addresser-use-1q-queues* t)
(quil::*safe-include-directory* (asdf:system-relative-pathname :cl-quil "tests/qasm-files/")))
(dolist (file (qasm-test-files))
(format t "| ~Va " 16 (trim-long-string (pathname-name file) 16))
(finish-output)
(handler-case
(let ((text (alexandria:read-file-into-string file)))
(tg:gc :full t)
(bordeaux-threads:with-timeout (timeout)
(with-stopwatch elapsed-time
(multiple-value-bind (cpp swaps)
(quil::compiler-hook (quil::parse text
:originating-file file)
chip
:protoquil t
:destructive t)
(format t "| ~Vf | ~Vd | ~Vd |~%"
8 (/ elapsed-time 1000)
5 swaps
8 (calculate-multiqubit-gate-depth (parsed-program-executable-code cpp)))))))
(bt:timeout ()
(format t "| ~8,'>d | ????? | ???????? |~%"
timeout)
(push (pathname-name file) timed-out)))
(finish-output)))
(let ((short-name (trim-long-string (pathname-name file) 16)))
(when (or (null named)
(if (atom named)
(string-equal named short-name)
(member short-name named :test 'string-equal)))
(format t "| ~Va " 16 short-name)
(finish-output)
(handler-case
(let ((text (alexandria:read-file-into-string file)))
(tg:gc :full t)
(bordeaux-threads:with-timeout (timeout)
(with-stopwatch elapsed-time
(multiple-value-bind (cpp swaps)
(quil::compiler-hook (quil::parse text
:originating-file file)
chip
:protoquil t
:destructive t)
(format t "| ~Vf | ~Vd | ~Vd |~%"
8 (/ elapsed-time 1000)
5 swaps
8 (calculate-multiqubit-gate-depth (parsed-program-executable-code cpp)))))))
(bt:timeout ()
(format t "| ~8,'>d | ????? | ???????? |~%"
timeout)
(push (pathname-name file) timed-out)))
(finish-output)))))
(print-rule)
(terpri)
(when timed-out
Expand Down
186 changes: 186 additions & 0 deletions benchmarking/quilc-mon-prof.lisp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
;;;; quilc-mon-prof.lisp
;;;;
;;;; Author: Mark David

(in-package #:cl-quil-benchmarking)



;;;; Monitoring and Profiling

;; This builds on quilc-perf.lisp and is for monitoring and profiling,
;; as opposed to simple benchmarks, and is a WIP in that we have not
;; established a makefile entry to run this. That is because, while
;; it's been useful to run for probing and experimenting in a REPL, we
;; so far lack really good theory of operation and associated modes of
;; running. We hope with time to get there.


(defparameter *monitor-types*
'(:mon #+sbcl :sb-sprof))

(defun monitor-run (program-type chip-type nq repeats monitor
&optional report-type sample-interval)
"Do REPEATS perf runs for PROGRAM-TYPE and CHIP-TYPE (as documented
at the top of QUILC-PERF module) using the specified MONITOR, which
must be one of those in the list *monitor-types*. Optional args
REPORT-TYPE and SAMPLE-INTERVAL are only relevant when MONITOR is
:SB-SPROF in which case REPORT-TYPE is passed as the same-named arg
to SB-SPROF:REPORT, and SAMPLE-INTERVAL is passed as the same-named
arg to SB-SPROF:START-PROFILING. Note as well that for :SB-SPROF
monitor, only the current thread is profiled. This returns no
useful value. It's just run in order to get output from the
monitor."
(when (not (member monitor *monitor-types*))
(unless (null monitor) ; if so, just silently default with no warning
(warn "unrecognized monitor, should be one of ~s; using ~s"
*monitor-types* (first *monitor-types*)))
(setq monitor (first *monitor-types*)))
(let (program chip)
(format t "~2%****~%Building ~a program (nQ = ~d)... " program-type nq)
(setq program (get-or-build-benchmark-program nq program-type))
(format t "DONE (~a).~%" program)
(format t "Building ~a chip (nQ = ~d)... " chip-type nq)
(setq chip (get-or-build-benchmark-chip nq chip-type))
(format t "DONE (~a).~%" chip)
(format t "** Doing ~d run~p... **~%" repeats repeats)
(let* ((*package* (find-package :cl-quil))
(thunk
#'(lambda ()
(dotimes (i repeats)
(prepare-environment-for-perf-run)
(format t "#~d: Compiling program/chip ... " i)
(let* ((t1 (get-internal-run-time))
(t2 (progn (do-one-quilc-perf-run program chip)
(get-internal-run-time)))
(elapsed-time (- t2 t1)))
(format
t
"DONE (~,2f sec compiler, ~2df sec real time w/overhead).~%"
;; -- 2nd timing is real time, as opposed to
;; internal run time. 'Overhead' is primarily GC +
;; warming.
(internal-time-to-seconds elapsed-time)
(internal-time-to-seconds (- t2 t1))))))))
(ecase monitor
(:mon (mon:monitor-form (funcall thunk)))
#+sbcl
(:sb-sprof
(progn
(sb-sprof:reset)
(sb-sprof:start-profiling
:sample-interval (or sample-interval 0.005) ; default = 0.01
:threads (list sb-thread:*current-thread*))
(funcall thunk)
(sb-sprof:report
:min-percent 3
:type (ecase report-type
((nil :flat) :flat)
(:graph :graph)))))))
(format t "** DONE with ~d runs. **~%" repeats)))

(defun do-monitor-runs (&key start step end
repeats monitor
report-type sam sample-interval)
(or repeats (setq repeats 3))
(loop :for nq := (or start 10)
:then (+ nq (or step 10))
:when (> nq (or end start))
:do (return)
:do (format t "~%**** NQ: ~d ****~%" nq)
(loop :for program-type :in *benchmark-program-types*
:do (loop :for chip-type :in *benchmark-chip-connectedness-types*
:do (monitor-run
program-type chip-type nq repeats monitor
report-type sample-interval)))))

;; Try this on SBCL: (do-monitor-runs :monitor ':sb-sprof)




(defun do-one (nq)
(do-one-nq-program-chip nq :hadamard :fully-connected))

(defparameter *min-sb-sprof-perf-pct* 1)

(defun do-one-nq-program-chip (nq program-type chip-type)
(let ((program (build-benchmark-program nq :hadamard))
(chip (build-benchmark-chip nq :fully-connected)))
(sb-sprof:reset)
(tg:gc :full t)
(sb-sprof:start-profiling :threads (list sb-thread:*current-thread*))
(time (benchmark-one-quilc-perf-run program chip))
(sb-sprof:stop-profiling)
(sb-sprof:report :type :graph :min-percent *min-sb-sprof-perf-pct*)
(sb-sprof:report :type :flat :min-percent *min-sb-sprof-perf-pct*)))

(defun do-one-mon (nq &optional program-type)
(let ((program (build-benchmark-program nq (or program-type :hadamard)))
(chip (build-benchmark-chip nq :fully-connected)))
(tg:gc :full t)
(sb-sprof:with-profiling (:max-samples 1000
:report :flat
:loop nil

:reset t
:sample-interval 0.01) ; default .01
(benchmark-one-quilc-perf-run program chip))))



;;;; SB-SPROF Runs of Various Flavors

(defparameter *default-sb-sprof-run-nqs*
'(20 50 80 110)
"Default list of nQ values for running various SB-SPROF calls below.")

(defun sb-sprof-run ()
(loop :for nq :in *default-sb-sprof-run-nqs*
:do (do-monitor-runs
:start nq
:monitor :sb-sprof
:repeats 1
:report-type :flat)))

(defun sb-sprof-graph ()
(loop :for nq :in *default-sb-sprof-run-nqs*
:do (do-monitor-runs
:start nq
:monitor :sb-sprof
:repeats 1
:report-type :flat)))

(defun sb-sprof-run-high-sample ()
(loop :for nq :in *default-sb-sprof-run-nqs*
:do (do-monitor-runs
:start nq
:monitor :sb-sprof
:repeats 1

:sample-interval 0.001 ; our default: 0.005
:report-type :flat)))

(defun sb-sprof-run-high-sample-plus-graph ()
(loop :for nq :in *default-sb-sprof-run-nqs*
:do (do-monitor-runs
:start nq
:monitor :sb-sprof
:repeats 1

:sample-interval 0.001 ; our default: 0.005
:report-type :graph))) ; our default: :flat

(defun sb-sprof-run-mon ()
(loop :for nq :in *default-sb-sprof-run-nqs*
:do (do-monitor-runs
:start nq
:monitor :mon
:repeats 1

:sample-interval 0.001 ; our default: 0.005
:report-type :graph))) ; our default: :flat

(defun one-monitor-run-bill-linear (nq)
(monitor-run :bell :linear nq 1 :mon :flat .0001))

Loading

0 comments on commit 93b3ba3

Please sign in to comment.