|
3828
|
1 /* |
|
|
2 |
|
|
3 This file is part of Octave. |
|
|
4 |
|
|
5 Octave is free software; you can redistribute it and/or modify it |
|
|
6 under the terms of the GNU General Public License as published by the |
|
|
7 Free Software Foundation; either version 2, or (at your option) any |
|
|
8 later version. |
|
|
9 |
|
|
10 Octave is distributed in the hope that it will be useful, but WITHOUT |
|
|
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
|
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
|
13 for more details. |
|
|
14 |
|
|
15 You should have received a copy of the GNU General Public License |
|
|
16 along with Octave; see the file COPYING. If not, write to the Free |
|
5307
|
17 Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA |
|
|
18 02110-1301, USA. |
|
3828
|
19 |
|
|
20 */ |
|
|
21 |
|
|
22 #ifdef HAVE_CONFIG_H |
|
|
23 #include <config.h> |
|
|
24 #endif |
|
|
25 |
|
4773
|
26 #if defined (HAVE_FFTW3) |
|
3828
|
27 |
|
4775
|
28 #include <iostream> |
|
|
29 #include <vector> |
|
|
30 |
|
4786
|
31 #include "lo-error.h" |
|
3828
|
32 #include "oct-fftw.h" |
|
4786
|
33 #include "quit.h" |
|
3828
|
34 |
|
4809
|
35 // Helper class to create and cache fftw plans for both 1d and |
|
|
36 // 2d. This implementation uses FFTW_ESTIMATE to create the plans, |
|
|
37 // which in theory is suboptimal, but provides quite reasonable |
|
|
38 // performance. |
|
4773
|
39 |
|
|
40 // Also note that if FFTW_ESTIMATE is not used the planner in FFTW3 |
|
4809
|
41 // destroys the input and output arrays. So with the form of the |
|
|
42 // current code we definitely want FFTW_ESTIMATE!! However, we use any |
|
|
43 // wsidom that is available, either in a FFTW3 system wide file or as |
|
|
44 // supplied by the user. |
|
4773
|
45 |
|
4809
|
46 // XXX FIXME XXX -- if we can ensure 16 byte alignment in Array<T> |
|
|
47 // (<T> *data) the FFTW3 can use SIMD instructions for further |
|
|
48 // acceleration. |
|
4773
|
49 |
|
4809
|
50 // Note that it is profitable to store the FFTW3 plans, for small |
|
|
51 // ffts. |
|
3828
|
52 |
|
|
53 class |
|
|
54 octave_fftw_planner |
|
|
55 { |
|
|
56 public: |
|
4808
|
57 |
|
|
58 octave_fftw_planner (void); |
|
3828
|
59 |
|
4773
|
60 fftw_plan create_plan (int dir, const int rank, const dim_vector dims, |
|
5275
|
61 octave_idx_type howmany, octave_idx_type stride, octave_idx_type dist, |
|
4773
|
62 const Complex *in, Complex *out); |
|
4808
|
63 |
|
4773
|
64 fftw_plan create_plan (const int rank, const dim_vector dims, |
|
5275
|
65 octave_idx_type howmany, octave_idx_type stride, octave_idx_type dist, |
|
4773
|
66 const double *in, Complex *out); |
|
3828
|
67 |
|
|
68 private: |
|
4808
|
69 |
|
3828
|
70 int plan_flags; |
|
|
71 |
|
4809
|
72 // XXX FIXME XXX -- perhaps this should be split into two classes? |
|
|
73 |
|
4773
|
74 // Plan for fft and ifft of complex values |
|
3828
|
75 fftw_plan plan[2]; |
|
4809
|
76 |
|
|
77 // dist |
|
5275
|
78 octave_idx_type d[2]; |
|
4809
|
79 |
|
|
80 // stride |
|
5275
|
81 octave_idx_type s[2]; |
|
4809
|
82 |
|
|
83 // rank |
|
|
84 int r[2]; |
|
|
85 |
|
|
86 // howmany |
|
5275
|
87 octave_idx_type h[2]; |
|
4809
|
88 |
|
|
89 // dims |
|
|
90 dim_vector n[2]; |
|
|
91 |
|
4808
|
92 bool simd_align[2]; |
|
5044
|
93 bool inplace[2]; |
|
4809
|
94 |
|
4773
|
95 // Plan for fft of real values |
|
|
96 fftw_plan rplan; |
|
4809
|
97 |
|
|
98 // dist |
|
5275
|
99 octave_idx_type rd; |
|
4809
|
100 |
|
|
101 // stride |
|
5275
|
102 octave_idx_type rs; |
|
4809
|
103 |
|
|
104 // rank |
|
|
105 int rr; |
|
|
106 |
|
|
107 // howmany |
|
5275
|
108 octave_idx_type rh; |
|
4809
|
109 |
|
|
110 // dims |
|
|
111 dim_vector rn; |
|
|
112 |
|
4808
|
113 bool rsimd_align; |
|
3828
|
114 }; |
|
|
115 |
|
4808
|
116 octave_fftw_planner::octave_fftw_planner (void) |
|
3828
|
117 { |
|
|
118 plan_flags = FFTW_ESTIMATE; |
|
|
119 |
|
|
120 plan[0] = plan[1] = 0; |
|
4773
|
121 d[0] = d[1] = s[0] = s[1] = r[0] = r[1] = h[0] = h[1] = 0; |
|
4808
|
122 simd_align[0] = simd_align[1] = false; |
|
5044
|
123 inplace[0] = inplace[1] = false; |
|
4808
|
124 n[0] = n[1] = dim_vector (); |
|
4773
|
125 |
|
|
126 rplan = 0; |
|
|
127 rd = rs = rr = rh = 0; |
|
4808
|
128 rsimd_align = false; |
|
4773
|
129 rn = dim_vector (); |
|
5044
|
130 |
|
4809
|
131 // If we have a system wide wisdom file, import it. |
|
4808
|
132 fftw_import_system_wisdom (); |
|
3828
|
133 } |
|
|
134 |
|
4808
|
135 #define CHECK_SIMD_ALIGNMENT(x) \ |
|
|
136 ((reinterpret_cast<ptrdiff_t> (x)) & 0xF == 0) |
|
|
137 |
|
3828
|
138 fftw_plan |
|
4773
|
139 octave_fftw_planner::create_plan (int dir, const int rank, |
|
5275
|
140 const dim_vector dims, octave_idx_type howmany, |
|
|
141 octave_idx_type stride, octave_idx_type dist, |
|
4773
|
142 const Complex *in, Complex *out) |
|
3828
|
143 { |
|
4773
|
144 int which = (dir == FFTW_FORWARD) ? 0 : 1; |
|
3828
|
145 fftw_plan *cur_plan_p = &plan[which]; |
|
|
146 bool create_new_plan = false; |
|
4808
|
147 bool ioalign = CHECK_SIMD_ALIGNMENT (in) && CHECK_SIMD_ALIGNMENT (out); |
|
5044
|
148 bool ioinplace = (in == out); |
|
3828
|
149 |
|
4809
|
150 // Don't create a new plan if we have a non SIMD plan already but |
|
|
151 // can do SIMD. This prevents endlessly recreating plans if we |
|
|
152 // change the alignment. |
|
|
153 |
|
4783
|
154 if (plan[which] == 0 || d[which] != dist || s[which] != stride |
|
5044
|
155 || r[which] != rank || h[which] != howmany |
|
|
156 || ioinplace != inplace[which] |
|
4808
|
157 || ((ioalign != simd_align[which]) ? !ioalign : false)) |
|
4773
|
158 create_new_plan = true; |
|
|
159 else |
|
4809
|
160 { |
|
|
161 // We still might not have the same shape of array. |
|
|
162 |
|
|
163 for (int i = 0; i < rank; i++) |
|
|
164 if (dims(i) != n[which](i)) |
|
|
165 { |
|
|
166 create_new_plan = true; |
|
|
167 break; |
|
|
168 } |
|
|
169 } |
|
3828
|
170 |
|
|
171 if (create_new_plan) |
|
|
172 { |
|
4773
|
173 d[which] = dist; |
|
|
174 s[which] = stride; |
|
|
175 r[which] = rank; |
|
|
176 h[which] = howmany; |
|
4808
|
177 simd_align[which] = ioalign; |
|
5044
|
178 inplace[which] = ioinplace; |
|
4773
|
179 n[which] = dims; |
|
|
180 |
|
4808
|
181 if (ioalign) |
|
|
182 plan_flags &= ~FFTW_UNALIGNED; |
|
|
183 else |
|
|
184 plan_flags |= FFTW_UNALIGNED; |
|
|
185 |
|
3828
|
186 if (*cur_plan_p) |
|
|
187 fftw_destroy_plan (*cur_plan_p); |
|
|
188 |
|
4809
|
189 // Note reversal of dimensions for column major storage in FFTW. |
|
|
190 |
|
4773
|
191 OCTAVE_LOCAL_BUFFER (int, tmp, rank); |
|
4809
|
192 |
|
4773
|
193 for (int i = 0, j = rank-1; i < rank; i++, j--) |
|
|
194 tmp[i] = dims(j); |
|
|
195 |
|
|
196 *cur_plan_p = |
|
|
197 fftw_plan_many_dft (rank, tmp, howmany, |
|
|
198 reinterpret_cast<fftw_complex *> (const_cast<Complex *> (in)), |
|
4774
|
199 0, stride, dist, reinterpret_cast<fftw_complex *> (out), |
|
|
200 0, stride, dist, dir, plan_flags); |
|
3828
|
201 |
|
|
202 if (*cur_plan_p == 0) |
|
|
203 (*current_liboctave_error_handler) ("Error creating fftw plan"); |
|
|
204 } |
|
|
205 |
|
|
206 return *cur_plan_p; |
|
|
207 } |
|
|
208 |
|
4773
|
209 fftw_plan |
|
|
210 octave_fftw_planner::create_plan (const int rank, const dim_vector dims, |
|
5275
|
211 octave_idx_type howmany, octave_idx_type stride, octave_idx_type dist, |
|
4773
|
212 const double *in, Complex *out) |
|
3828
|
213 { |
|
4773
|
214 fftw_plan *cur_plan_p = &rplan; |
|
3828
|
215 bool create_new_plan = false; |
|
4808
|
216 bool ioalign = CHECK_SIMD_ALIGNMENT (in) && CHECK_SIMD_ALIGNMENT (out); |
|
3828
|
217 |
|
4809
|
218 // Don't create a new plan if we have a non SIMD plan already but |
|
|
219 // can do SIMD. This prevents endlessly recreating plans if we |
|
|
220 // change the alignment. |
|
|
221 |
|
4783
|
222 if (rplan == 0 || rd != dist || rs != stride || rr != rank |
|
4808
|
223 || rh != howmany || ((ioalign != rsimd_align) ? !ioalign : false)) |
|
4773
|
224 create_new_plan = true; |
|
|
225 else |
|
4809
|
226 { |
|
|
227 // We still might not have the same shape of array. |
|
|
228 |
|
|
229 for (int i = 0; i < rank; i++) |
|
|
230 if (dims(i) != rn(i)) |
|
|
231 { |
|
|
232 create_new_plan = true; |
|
|
233 break; |
|
|
234 } |
|
|
235 } |
|
3828
|
236 |
|
|
237 if (create_new_plan) |
|
|
238 { |
|
4773
|
239 rd = dist; |
|
|
240 rs = stride; |
|
|
241 rr = rank; |
|
|
242 rh = howmany; |
|
4808
|
243 rsimd_align = ioalign; |
|
4773
|
244 rn = dims; |
|
|
245 |
|
4808
|
246 if (ioalign) |
|
|
247 plan_flags &= ~FFTW_UNALIGNED; |
|
|
248 else |
|
|
249 plan_flags |= FFTW_UNALIGNED; |
|
|
250 |
|
3828
|
251 if (*cur_plan_p) |
|
4773
|
252 fftw_destroy_plan (*cur_plan_p); |
|
3828
|
253 |
|
4809
|
254 // Note reversal of dimensions for column major storage in FFTW. |
|
|
255 |
|
4773
|
256 OCTAVE_LOCAL_BUFFER (int, tmp, rank); |
|
4809
|
257 |
|
4773
|
258 for (int i = 0, j = rank-1; i < rank; i++, j--) |
|
|
259 tmp[i] = dims(j); |
|
|
260 |
|
|
261 *cur_plan_p = |
|
|
262 fftw_plan_many_dft_r2c (rank, tmp, howmany, |
|
|
263 (const_cast<double *> (in)), |
|
4774
|
264 0, stride, dist, reinterpret_cast<fftw_complex *> (out), |
|
|
265 0, stride, dist, plan_flags); |
|
3828
|
266 |
|
|
267 if (*cur_plan_p == 0) |
|
4773
|
268 (*current_liboctave_error_handler) ("Error creating fftw plan"); |
|
3828
|
269 } |
|
|
270 |
|
|
271 return *cur_plan_p; |
|
|
272 } |
|
|
273 |
|
|
274 static octave_fftw_planner fftw_planner; |
|
|
275 |
|
4775
|
276 static inline void |
|
|
277 convert_packcomplex_1d (Complex *out, size_t nr, size_t nc, |
|
5275
|
278 octave_idx_type stride, octave_idx_type dist) |
|
4773
|
279 { |
|
4785
|
280 OCTAVE_QUIT; |
|
|
281 |
|
|
282 // Fill in the missing data. |
|
|
283 |
|
4773
|
284 for (size_t i = 0; i < nr; i++) |
|
|
285 for (size_t j = nc/2+1; j < nc; j++) |
|
|
286 out[j*stride + i*dist] = conj(out[(nc - j)*stride + i*dist]); |
|
4785
|
287 |
|
|
288 OCTAVE_QUIT; |
|
4773
|
289 } |
|
|
290 |
|
4775
|
291 static inline void |
|
|
292 convert_packcomplex_Nd (Complex *out, const dim_vector &dv) |
|
3828
|
293 { |
|
4773
|
294 size_t nc = dv(0); |
|
|
295 size_t nr = dv(1); |
|
4808
|
296 size_t np = (dv.length () > 2 ? dv.numel () / nc / nr : 1); |
|
4773
|
297 size_t nrp = nr * np; |
|
|
298 Complex *ptr1, *ptr2; |
|
|
299 |
|
4785
|
300 OCTAVE_QUIT; |
|
|
301 |
|
|
302 // Create space for the missing elements. |
|
|
303 |
|
4773
|
304 for (size_t i = 0; i < nrp; i++) |
|
|
305 { |
|
|
306 ptr1 = out + i * (nc/2 + 1) + nrp*((nc-1)/2); |
|
|
307 ptr2 = out + i * nc; |
|
|
308 for (size_t j = 0; j < nc/2+1; j++) |
|
|
309 *ptr2++ = *ptr1++; |
|
|
310 } |
|
|
311 |
|
4785
|
312 OCTAVE_QUIT; |
|
|
313 |
|
|
314 // Fill in the missing data for the rank = 2 case directly for speed. |
|
|
315 |
|
4773
|
316 for (size_t i = 0; i < np; i++) |
|
|
317 { |
|
|
318 for (size_t j = 1; j < nr; j++) |
|
|
319 for (size_t k = nc/2+1; k < nc; k++) |
|
|
320 out[k + (j + i*nr)*nc] = conj(out[nc - k + ((i+1)*nr - j)*nc]); |
|
|
321 |
|
|
322 for (size_t j = nc/2+1; j < nc; j++) |
|
|
323 out[j + i*nr*nc] = conj(out[(i*nr+1)*nc - j]); |
|
|
324 } |
|
|
325 |
|
4785
|
326 OCTAVE_QUIT; |
|
|
327 |
|
|
328 // Now do the permutations needed for rank > 2 cases. |
|
|
329 |
|
4773
|
330 size_t jstart = dv(0) * dv(1); |
|
|
331 size_t kstep = dv(0); |
|
|
332 size_t nel = dv.numel (); |
|
4785
|
333 |
|
4808
|
334 for (int inner = 2; inner < dv.length (); inner++) |
|
4773
|
335 { |
|
|
336 size_t jmax = jstart * dv(inner); |
|
|
337 for (size_t i = 0; i < nel; i+=jmax) |
|
|
338 for (size_t j = jstart, jj = jmax-jstart; j < jj; |
|
|
339 j+=jstart, jj-=jstart) |
|
|
340 for (size_t k = 0; k < jstart; k+= kstep) |
|
|
341 for (size_t l = nc/2+1; l < nc; l++) |
|
|
342 { |
|
|
343 Complex tmp = out[i+ j + k + l]; |
|
|
344 out[i + j + k + l] = out[i + jj + k + l]; |
|
|
345 out[i + jj + k + l] = tmp; |
|
|
346 } |
|
|
347 jstart = jmax; |
|
|
348 } |
|
4785
|
349 |
|
|
350 OCTAVE_QUIT; |
|
4773
|
351 } |
|
|
352 |
|
|
353 int |
|
|
354 octave_fftw::fft (const double *in, Complex *out, size_t npts, |
|
5275
|
355 size_t nsamples, octave_idx_type stride, octave_idx_type dist) |
|
4773
|
356 { |
|
|
357 dist = (dist < 0 ? npts : dist); |
|
|
358 |
|
|
359 dim_vector dv (npts); |
|
|
360 fftw_plan plan = fftw_planner.create_plan (1, dv, nsamples, stride, dist, |
|
|
361 in, out); |
|
|
362 |
|
|
363 fftw_execute_dft_r2c (plan, (const_cast<double *>(in)), |
|
|
364 reinterpret_cast<fftw_complex *> (out)); |
|
|
365 |
|
4809
|
366 // Need to create other half of the transform. |
|
|
367 |
|
4773
|
368 convert_packcomplex_1d (out, nsamples, npts, stride, dist); |
|
3828
|
369 |
|
|
370 return 0; |
|
|
371 } |
|
|
372 |
|
|
373 int |
|
4773
|
374 octave_fftw::fft (const Complex *in, Complex *out, size_t npts, |
|
5275
|
375 size_t nsamples, octave_idx_type stride, octave_idx_type dist) |
|
3828
|
376 { |
|
4773
|
377 dist = (dist < 0 ? npts : dist); |
|
|
378 |
|
|
379 dim_vector dv (npts); |
|
|
380 fftw_plan plan = fftw_planner.create_plan (FFTW_FORWARD, 1, dv, nsamples, |
|
|
381 stride, dist, in, out); |
|
|
382 |
|
|
383 fftw_execute_dft (plan, |
|
|
384 reinterpret_cast<fftw_complex *> (const_cast<Complex *>(in)), |
|
|
385 reinterpret_cast<fftw_complex *> (out)); |
|
|
386 |
|
|
387 return 0; |
|
|
388 } |
|
|
389 |
|
|
390 int |
|
|
391 octave_fftw::ifft (const Complex *in, Complex *out, size_t npts, |
|
5275
|
392 size_t nsamples, octave_idx_type stride, octave_idx_type dist) |
|
4773
|
393 { |
|
|
394 dist = (dist < 0 ? npts : dist); |
|
|
395 |
|
|
396 dim_vector dv (npts); |
|
|
397 fftw_plan plan = fftw_planner.create_plan (FFTW_BACKWARD, 1, dv, nsamples, |
|
|
398 stride, dist, in, out); |
|
|
399 |
|
|
400 fftw_execute_dft (plan, |
|
|
401 reinterpret_cast<fftw_complex *> (const_cast<Complex *>(in)), |
|
|
402 reinterpret_cast<fftw_complex *> (out)); |
|
3828
|
403 |
|
|
404 const Complex scale = npts; |
|
4773
|
405 for (size_t j = 0; j < nsamples; j++) |
|
|
406 for (size_t i = 0; i < npts; i++) |
|
|
407 out[i*stride + j*dist] /= scale; |
|
3828
|
408 |
|
|
409 return 0; |
|
|
410 } |
|
|
411 |
|
|
412 int |
|
4773
|
413 octave_fftw::fftNd (const double *in, Complex *out, const int rank, |
|
|
414 const dim_vector &dv) |
|
3828
|
415 { |
|
5275
|
416 octave_idx_type dist = 1; |
|
4773
|
417 for (int i = 0; i < rank; i++) |
|
|
418 dist *= dv(i); |
|
|
419 |
|
|
420 // Fool with the position of the start of the output matrix, so that |
|
4809
|
421 // creating other half of the matrix won't cause cache problems. |
|
|
422 |
|
5275
|
423 octave_idx_type offset = (dv.numel () / dv(0)) * ((dv(0) - 1) / 2); |
|
4773
|
424 |
|
|
425 fftw_plan plan = fftw_planner.create_plan (rank, dv, 1, 1, dist, |
|
|
426 in, out + offset); |
|
|
427 |
|
|
428 fftw_execute_dft_r2c (plan, (const_cast<double *>(in)), |
|
|
429 reinterpret_cast<fftw_complex *> (out+ offset)); |
|
|
430 |
|
4809
|
431 // Need to create other half of the transform. |
|
|
432 |
|
4773
|
433 convert_packcomplex_Nd (out, dv); |
|
3828
|
434 |
|
|
435 return 0; |
|
|
436 } |
|
|
437 |
|
|
438 int |
|
4773
|
439 octave_fftw::fftNd (const Complex *in, Complex *out, const int rank, |
|
|
440 const dim_vector &dv) |
|
3828
|
441 { |
|
5275
|
442 octave_idx_type dist = 1; |
|
4773
|
443 for (int i = 0; i < rank; i++) |
|
|
444 dist *= dv(i); |
|
|
445 |
|
|
446 fftw_plan plan = fftw_planner.create_plan (FFTW_FORWARD, rank, dv, 1, 1, |
|
|
447 dist, in, out); |
|
|
448 |
|
|
449 fftw_execute_dft (plan, |
|
|
450 reinterpret_cast<fftw_complex *> (const_cast<Complex *>(in)), |
|
|
451 reinterpret_cast<fftw_complex *> (out)); |
|
|
452 |
|
|
453 return 0; |
|
|
454 } |
|
3828
|
455 |
|
4773
|
456 int |
|
|
457 octave_fftw::ifftNd (const Complex *in, Complex *out, const int rank, |
|
4784
|
458 const dim_vector &dv) |
|
4773
|
459 { |
|
5275
|
460 octave_idx_type dist = 1; |
|
4773
|
461 for (int i = 0; i < rank; i++) |
|
|
462 dist *= dv(i); |
|
|
463 |
|
|
464 fftw_plan plan = fftw_planner.create_plan (FFTW_BACKWARD, rank, dv, 1, 1, |
|
|
465 dist, in, out); |
|
|
466 |
|
|
467 fftw_execute_dft (plan, |
|
|
468 reinterpret_cast<fftw_complex *> (const_cast<Complex *>(in)), |
|
|
469 reinterpret_cast<fftw_complex *> (out)); |
|
|
470 |
|
|
471 const size_t npts = dv.numel (); |
|
3828
|
472 const Complex scale = npts; |
|
|
473 for (size_t i = 0; i < npts; i++) |
|
4773
|
474 out[i] /= scale; |
|
3828
|
475 |
|
|
476 return 0; |
|
|
477 } |
|
|
478 |
|
|
479 #endif |
|
|
480 |
|
|
481 /* |
|
|
482 ;;; Local Variables: *** |
|
|
483 ;;; mode: C++ *** |
|
|
484 ;;; End: *** |
|
|
485 */ |
|
|
486 |
