Mercurial > hg > octave-lyh
annotate scripts/testfun/speed.m @ 8202:cf59d542f33e
replace all TODOs and XXXs with FIXMEs
| author | Jaroslav Hajek <highegg@gmail.com> |
|---|---|
| date | Wed, 08 Oct 2008 20:00:25 +0200 |
| parents | 3422f39573b1 |
| children | bc982528de11 |
| rev | line source |
|---|---|
| 7017 | 1 ## Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007 |
| 2 ## Paul Kienzle | |
| 7016 | 3 ## |
| 4 ## This file is part of Octave. | |
| 5589 | 5 ## |
| 7016 | 6 ## Octave is free software; you can redistribute it and/or modify it |
| 7 ## under the terms of the GNU General Public License as published by | |
| 8 ## the Free Software Foundation; either version 3 of the License, or (at | |
| 9 ## your option) any later version. | |
| 5589 | 10 ## |
| 7016 | 11 ## Octave is distributed in the hope that it will be useful, but |
| 12 ## WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 13 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| 14 ## General Public License for more details. | |
| 5589 | 15 ## |
| 16 ## You should have received a copy of the GNU General Public License | |
| 7016 | 17 ## along with Octave; see the file COPYING. If not, see |
| 18 ## <http://www.gnu.org/licenses/>. | |
| 5589 | 19 |
| 20 ## -*- texinfo -*- | |
| 5798 | 21 ## @deftypefn {Function File} {} speed (@var{f}, @var{init}, @var{max_n}, @var{f2}, @var{tol}) |
| 22 ## @deftypefnx {Function File} {[@var{order}, @var{n}, @var{T_f}, @var{T_f2}] =} speed (@dots{}) | |
| 5589 | 23 ## |
| 24 ## Determine the execution time of an expression for various @var{n}. | |
| 25 ## The @var{n} are log-spaced from 1 to @var{max_n}. For each @var{n}, | |
| 26 ## an initialization expression is computed to create whatever data | |
| 5798 | 27 ## are needed for the test. If a second expression is given, the |
| 28 ## execution times of the two expressions will be compared. Called | |
| 29 ## without output arguments the results are presented graphically. | |
| 5589 | 30 ## |
| 31 ## @table @code | |
| 32 ## @item @var{f} | |
| 33 ## The expression to evaluate. | |
| 34 ## | |
| 35 ## @item @var{max_n} | |
| 5798 | 36 ## The maximum test length to run. Default value is 100. Alternatively, |
| 37 ## use @code{[min_n,max_n]} or for complete control, @code{[n1,n2,@dots{},nk]}. | |
| 5589 | 38 ## |
| 39 ## @item @var{init} | |
| 40 ## Initialization expression for function argument values. Use @var{k} | |
| 41 ## for the test number and @var{n} for the size of the test. This should | |
| 42 ## compute values for all variables listed in args. Note that init will | |
| 43 ## be evaluated first for k=0, so things which are constant throughout | |
| 44 ## the test can be computed then. The default value is @code{@var{x} = | |
| 45 ## randn (@var{n}, 1);}. | |
| 46 ## | |
| 47 ## @item @var{f2} | |
| 48 ## An alternative expression to evaluate, so the speed of the two | |
| 49 ## can be compared. Default is @code{[]}. | |
| 50 ## | |
| 51 ## @item @var{tol} | |
| 52 ## If @var{tol} is @code{Inf}, then no comparison will be made between the | |
| 53 ## results of expression @var{f} and expression @var{f2}. Otherwise, | |
| 54 ## expression @var{f} should produce a value @var{v} and expression @var{f2} | |
| 55 ## should produce a value @var{v2}, and these shall be compared using | |
| 6429 | 56 ## @code{assert(@var{v},@var{v2},@var{tol})}. If @var{tol} is positive, |
| 7001 | 57 ## the tolerance is assumed to be absolute. If @var{tol} is negative, |
| 6429 | 58 ## the tolerance is assumed to be relative. The default is @code{eps}. |
| 5589 | 59 ## |
| 5798 | 60 ## @item @var{order} |
| 61 ## The time complexity of the expression @code{O(a n^p)}. This | |
| 62 ## is a structure with fields @code{a} and @code{p}. | |
| 5589 | 63 ## |
| 5798 | 64 ## @item @var{n} |
| 7001 | 65 ## The values @var{n} for which the expression was calculated and |
| 5798 | 66 ## the execution time was greater than zero. |
| 5589 | 67 ## |
| 5798 | 68 ## @item @var{T_f} |
| 69 ## The nonzero execution times recorded for the expression @var{f} in seconds. | |
| 70 ## | |
| 71 ## @item @var{T_f2} | |
| 72 ## The nonzero execution times recorded for the expression @var{f2} in seconds. | |
| 73 ## If it is needed, the mean time ratio is just @code{mean(T_f./T_f2)}. | |
| 74 ## | |
| 5589 | 75 ## @end table |
| 76 ## | |
| 5798 | 77 ## The slope of the execution time graph shows the approximate |
| 78 ## power of the asymptotic running time @code{O(n^p)}. This | |
| 79 ## power is plotted for the region over which it is approximated | |
| 80 ## (the latter half of the graph). The estimated power is not | |
| 81 ## very accurate, but should be sufficient to determine the | |
| 82 ## general order of your algorithm. It should indicate if for | |
| 83 ## example your implementation is unexpectedly @code{O(n^2)} | |
| 84 ## rather than @code{O(n)} because it extends a vector each | |
| 85 ## time through the loop rather than preallocating one which is | |
| 86 ## big enough. For example, in the current version of Octave, | |
| 87 ## the following is not the expected @code{O(n)}: | |
| 5589 | 88 ## |
| 5798 | 89 ## @example |
| 90 ## speed("for i=1:n,y@{i@}=x(i); end", "", [1000,10000]) | |
| 91 ## @end example | |
| 92 ## | |
| 93 ## but it is if you preallocate the cell array @code{y}: | |
| 5589 | 94 ## |
| 95 ## @example | |
| 5798 | 96 ## speed("for i=1:n,y@{i@}=x(i);end", ... |
| 97 ## "x=rand(n,1);y=cell(size(x));", [1000,10000]) | |
| 98 ## @end example | |
| 99 ## | |
| 100 ## An attempt is made to approximate the cost of the individual | |
| 101 ## operations, but it is wildly inaccurate. You can improve the | |
| 102 ## stability somewhat by doing more work for each @code{n}. For | |
| 103 ## example: | |
| 104 ## | |
| 105 ## @example | |
| 106 ## speed("airy(x)", "x=rand(n,10)", [10000,100000]) | |
| 5589 | 107 ## @end example |
| 108 ## | |
| 5798 | 109 ## When comparing a new and original expression, the line on the |
| 110 ## speedup ratio graph should be larger than 1 if the new expression | |
| 111 ## is faster. Better algorithms have a shallow slope. Generally, | |
| 112 ## vectorizing an algorithm will not change the slope of the execution | |
| 113 ## time graph, but it will shift it relative to the original. For | |
| 114 ## example: | |
| 115 ## | |
| 116 ## @example | |
| 117 ## speed("v=sum(x)", "", [10000,100000], ... | |
| 118 ## "v=0;for i=1:length(x),v+=x(i);end") | |
| 119 ## @end example | |
| 120 ## | |
| 5589 | 121 ## A more complex example, if you had an original version of @code{xcorr} |
| 122 ## using for loops and another version using an FFT, you could compare the | |
| 123 ## run speed for various lags as follows, or for a fixed lag with varying | |
| 124 ## vector lengths as follows: | |
| 125 ## | |
| 126 ## @example | |
| 127 ## speed("v=xcorr(x,n)", "x=rand(128,1);", 100, ... | |
| 6429 | 128 ## "v2=xcorr_orig(x,n)", -100*eps) |
| 5589 | 129 ## speed("v=xcorr(x,15)", "x=rand(20+n,1);", 100, ... |
| 6429 | 130 ## "v2=xcorr_orig(x,n)", -100*eps) |
| 5589 | 131 ## @end example |
| 132 ## | |
| 7001 | 133 ## Assuming one of the two versions is in @var{xcorr_orig}, this |
| 5589 | 134 ## would compare their speed and their output values. Note that the |
| 135 ## FFT version is not exact, so we specify an acceptable tolerance on | |
| 136 ## the comparison @code{100*eps}, and the errors should be computed | |
| 137 ## relatively, as @code{abs((@var{x} - @var{y})./@var{y})} rather than | |
| 138 ## absolutely as @code{abs(@var{x} - @var{y})}. | |
| 139 ## | |
| 140 ## Type @code{example('speed')} to see some real examples. Note for | |
| 141 ## obscure reasons, you can't run examples 1 and 2 directly using | |
| 142 ## @code{demo('speed')}. Instead use, @code{eval(example('speed',1))} | |
| 143 ## and @code{eval(example('speed',2))}. | |
| 144 ## @end deftypefn | |
| 145 | |
|
8202
cf59d542f33e
replace all TODOs and XXXs with FIXMEs
Jaroslav Hajek <highegg@gmail.com>
parents:
7540
diff
changeset
|
146 ## FIXME: consider two dimensional speedup surfaces for functions like kron. |
| 5798 | 147 function [__order, __test_n, __tnew, __torig] ... |
| 6494 | 148 = speed (__f1, __init, __max_n, __f2, __tol) |
| 149 | |
| 150 if (nargin < 1 || nargin > 6) | |
| 6046 | 151 print_usage (); |
| 5589 | 152 endif |
| 6494 | 153 |
| 154 if (nargin < 2 || isempty (__init)) | |
| 5589 | 155 __init = "x = randn(n, 1);"; |
| 156 endif | |
| 6494 | 157 |
| 158 if (nargin < 3 || isempty (__max_n)) | |
| 159 __max_n = 100; | |
| 160 endif | |
| 161 | |
| 162 if (nargin < 4) | |
| 163 __f2 = []; | |
| 164 endif | |
| 165 | |
| 166 if (nargin < 5 || isempty (__tol)) | |
| 167 __tol = eps; | |
| 168 endif | |
| 5798 | 169 |
| 170 __numtests = 15; | |
| 5589 | 171 |
| 5798 | 172 ## Let user specify range of n |
| 6494 | 173 if (isscalar (__max_n)) |
| 5798 | 174 __min_n = 1; |
| 6494 | 175 assert (__max_n > __min_n); |
| 176 __test_n = logspace (0, log10 (__max_n), __numtests); | |
| 177 elseif (length (__max_n) == 2) | |
| 5798 | 178 __min_n = __max_n(1); |
| 179 __max_n = __max_n(2); | |
| 6494 | 180 assert (__min_n >= 1); |
| 181 __test_n = logspace (log10 (__min_n), log10 (__max_n), __numtests); | |
| 5798 | 182 else |
| 183 __test_n = __max_n; | |
| 184 endif | |
| 6494 | 185 __test_n = unique (round (__test_n)); # Force n to be an integer |
| 186 assert (__test_n >= 1); | |
| 5589 | 187 |
| 6494 | 188 __torig = __tnew = zeros (size (__test_n)); |
| 5589 | 189 |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
190 disp (cstrcat ("testing ", __f1, "\ninit: ", __init)); |
| 5589 | 191 |
| 192 ## make sure the functions are freshly loaded by evaluating them at | |
| 5798 | 193 ## test_n(1); first have to initialize the args though. |
| 6494 | 194 n = 1; |
| 195 k = 0; | |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
196 eval (cstrcat (__init, ";")); |
| 6494 | 197 if (! isempty (__f2)) |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
198 eval (cstrcat (__f2, ";")); |
| 6494 | 199 endif |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
200 eval (cstrcat (__f1, ";")); |
| 5589 | 201 |
| 202 ## run the tests | |
| 6494 | 203 for k = 1:length (__test_n) |
| 204 n = __test_n(k); | |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
205 eval (cstrcat (__init, ";")); |
| 5589 | 206 |
| 6494 | 207 printf ("n%i=%i ",k, n); |
| 208 fflush (stdout); | |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
209 eval (cstrcat ("__t=time();", __f1, "; __v1=ans; __t = time()-__t;")); |
| 5589 | 210 if (__t < 0.25) |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
211 eval (cstrcat ("__t2=time();", __f1, "; __t2 = time()-__t2;")); |
|
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
212 eval (cstrcat ("__t3=time();", __f1, "; __t3 = time()-__t3;")); |
| 6494 | 213 __t = min ([__t, __t2, __t3]); |
| 5589 | 214 endif |
| 215 __tnew(k) = __t; | |
| 216 | |
| 6494 | 217 if (! isempty (__f2)) |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
218 eval (cstrcat ("__t=time();", __f2, "; __v2=ans; __t = time()-__t;")); |
| 5589 | 219 if (__t < 0.25) |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
220 eval (cstrcat ("__t2=time();", __f2, "; __t2 = time()-__t2;")); |
|
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
221 eval (cstrcat ("__t3=time();", __f2, "; __t3 = time()-__t3;")); |
| 5589 | 222 endif |
| 223 __torig(k) = __t; | |
| 6494 | 224 if (! isinf(__tol)) |
| 225 assert (__v1, __v2, __tol); | |
| 5589 | 226 endif |
| 227 endif | |
| 5798 | 228 endfor |
| 5589 | 229 |
| 5798 | 230 ## Drop times of zero |
| 6494 | 231 if (! isempty (__f2)) |
| 232 zidx = (__tnew < 100*eps | __torig < 100*eps); | |
| 5798 | 233 __test_n(zidx) = []; |
| 234 __tnew(zidx) = []; | |
| 235 __torig(zidx) = []; | |
| 5589 | 236 else |
| 6494 | 237 zidx = (__tnew < 100*eps); |
| 5798 | 238 __test_n(zidx) = []; |
| 239 __tnew(zidx) = []; | |
| 5589 | 240 endif |
| 6494 | 241 |
| 5798 | 242 ## Approximate time complexity and return it if requested |
| 6494 | 243 tailidx = ceil(length(__test_n)/2):length(__test_n); |
| 244 p = polyfit (log (__test_n(tailidx)), log (__tnew(tailidx)), 1); | |
| 245 if (nargout > 0) | |
| 5798 | 246 __order.p = p(1); |
| 6494 | 247 __order.a = exp (p(2)); |
| 5798 | 248 endif |
| 5589 | 249 |
| 5798 | 250 ## Plot the data if no output is requested. |
| 251 doplot = (nargout == 0); | |
| 6430 | 252 |
| 253 if (doplot) | |
| 254 figure; | |
| 255 endif | |
| 5798 | 256 |
| 6494 | 257 if (doplot && ! isempty (__f2)) |
| 258 subplot (1, 2, 1); | |
| 259 semilogx (__test_n, __torig./__tnew, | |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
260 cstrcat ("-*r;", strrep (__f1, ";", "."), "/", |
| 6494 | 261 strrep (__f2, ";", "."), ";"), |
| 5798 | 262 __test_n, __tnew./__torig, |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
263 cstrcat ("-*g;", strrep (__f2, ";", "."), "/", |
| 6494 | 264 strrep (__f1, ";", "."), ";")); |
| 265 xlabel ("test length"); | |
| 266 title (__f1); | |
| 267 ylabel ("speedup ratio"); | |
| 268 | |
| 269 subplot (1, 2, 2); | |
| 270 loglog (__test_n, __tnew*1000, | |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
271 cstrcat ("*-g;", strrep (__f1, ";", "."), ";" ), |
| 6494 | 272 __test_n, __torig*1000, |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
273 cstrcat ("*-r;", strrep (__f2,";","."), ";")); |
| 6494 | 274 |
| 275 xlabel ("test length"); | |
| 5589 | 276 ylabel ("best execution time (ms)"); |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
277 title (cstrcat ("init: ", __init)); |
| 6494 | 278 |
| 5798 | 279 ratio = mean (__torig ./ __tnew); |
| 6494 | 280 printf ("\n\nMean runtime ratio = %.3g for '%s' vs '%s'\n", |
| 5798 | 281 ratio, __f2, __f1); |
| 282 | |
| 6494 | 283 elseif (doplot) |
| 5798 | 284 |
| 6494 | 285 loglog (__test_n, __tnew*1000, "*-g;execution time;"); |
| 286 xlabel ("test length"); | |
| 5589 | 287 ylabel ("best execution time (ms)"); |
|
7540
3422f39573b1
strcat.m: Matlab compatibility, with cstrcat.m replacing conventional strcat.m.
Ben Abbott <bpabbott@mac.com>
parents:
7017
diff
changeset
|
288 title (cstrcat (__f1, " init: ", __init)); |
| 5798 | 289 |
| 5589 | 290 endif |
| 5798 | 291 |
| 6494 | 292 if (doplot) |
| 5798 | 293 |
| 294 ## Plot time complexity approximation (using milliseconds). | |
| 6494 | 295 order = sprintf ("O(n^%g)", round (10*p(1))/10); |
| 296 v = polyval (p, log (__test_n(tailidx))); | |
| 297 | |
| 298 loglog (__test_n(tailidx), exp(v)*1000, sprintf ("b;%s;", order)); | |
| 5798 | 299 |
| 300 ## Get base time to 1 digit of accuracy | |
| 6494 | 301 dt = exp (p(2)); |
| 302 dt = floor (dt/10^floor(log10(dt)))*10^floor(log10(dt)); | |
| 303 if (log10 (dt) >= -0.5) | |
| 304 time = sprintf ("%g s", dt); | |
| 305 elseif (log10 (dt) >= -3.5) | |
| 306 time = sprintf ("%g ms", dt*1e3); | |
| 307 elseif (log10 (dt) >= -6.5) | |
| 308 time = sprintf ("%g us", dt*1e6); | |
| 309 else | |
| 310 time = sprintf ("%g ns", dt*1e9); | |
| 5798 | 311 endif |
| 312 | |
| 313 ## Display nicely formatted complexity. | |
| 6494 | 314 printf ("\nFor %s:\n", __f1); |
| 5798 | 315 printf (" asymptotic power: %s\n", order); |
| 316 printf (" approximate time per operation: %s\n", time); | |
| 317 | |
| 318 endif | |
| 319 | |
| 5589 | 320 endfunction |
| 321 | |
| 322 %!demo if 1 | |
| 323 %! function x = build_orig(n) | |
| 324 %! ## extend the target vector on the fly | |
| 325 %! for i=0:n-1, x([1:10]+i*10) = 1:10; endfor | |
| 326 %! endfunction | |
| 327 %! function x = build(n) | |
| 328 %! ## preallocate the target vector | |
| 329 %! x = zeros(1, n*10); | |
| 330 %! try | |
| 331 %! if (prefer_column_vectors), x = x.'; endif | |
| 332 %! catch | |
| 333 %! end | |
| 334 %! for i=0:n-1, x([1:10]+i*10) = 1:10; endfor | |
| 335 %! endfunction | |
| 336 %! | |
| 337 %! disp("-----------------------"); | |
| 338 %! type build_orig; | |
| 339 %! disp("-----------------------"); | |
| 340 %! type build; | |
| 341 %! disp("-----------------------"); | |
| 342 %! | |
| 343 %! disp("Preallocated vector test.\nThis takes a little while..."); | |
| 6429 | 344 %! speed('build(n)', '', 1000, 'build_orig(n)'); |
| 5589 | 345 %! clear build build_orig |
| 346 %! disp("Note how much faster it is to pre-allocate a vector."); | |
| 347 %! disp("Notice the peak speedup ratio."); | |
| 348 %! endif | |
| 349 | |
| 350 %!demo if 1 | |
| 351 %! function x = build_orig(n) | |
| 352 %! for i=0:n-1, x([1:10]+i*10) = 1:10; endfor | |
| 353 %! endfunction | |
| 354 %! function x = build(n) | |
| 355 %! idx = [1:10]'; | |
| 356 %! x = idx(:,ones(1,n)); | |
| 357 %! x = reshape(x, 1, n*10); | |
| 358 %! try | |
| 359 %! if (prefer_column_vectors), x = x.'; endif | |
| 360 %! catch | |
| 361 %! end | |
| 362 %! endfunction | |
| 363 %! | |
| 364 %! disp("-----------------------"); | |
| 365 %! type build_orig; | |
| 366 %! disp("-----------------------"); | |
| 367 %! type build; | |
| 368 %! disp("-----------------------"); | |
| 369 %! | |
| 370 %! disp("Vectorized test. This takes a little while..."); | |
| 6429 | 371 %! speed('build(n)', '', 1000, 'build_orig(n)'); |
| 5589 | 372 %! clear build build_orig |
| 373 %! disp("-----------------------"); | |
| 374 %! disp("This time, the for loop is done away with entirely."); | |
| 375 %! disp("Notice how much bigger the speedup is then in example 1."); | |
| 376 %! endif |