exec_preprocessingΒΆ
This section contains the exec_preprocessing script.
Download file: exec_preprocessing.py
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 | ###########################################################################
# (C) 2016 Elettra - Sincrotrone Trieste S.C.p.A.. All rights reserved. #
# #
# #
# This file is part of STP-Core, the Python core of SYRMEP Tomo Project, #
# a software tool for the reconstruction of experimental CT datasets. #
# #
# STP-Core is free software: you can redistribute it and/or modify it #
# under the terms of the GNU General Public License as published by the #
# Free Software Foundation, either version 3 of the License, or (at your #
# option) any later version. #
# #
# STP-Core is distributed in the hope that it will be useful, but WITHOUT #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License #
# for more details. #
# #
# You should have received a copy of the GNU General Public License #
# along with STP-Core. If not, see <http://www.gnu.org/licenses/>. #
# #
###########################################################################
#
# Author: Francesco Brun
# Last modified: August, 8th 2016
#
from sys import argv, exit
from os import remove, sep, linesep
from os.path import exists
from numpy import float32, amin, amax, isscalar
from time import time
from multiprocessing import Process, Lock
# pystp-specific:
from stp_core.preprocess.extfov_correction import extfov_correction
from stp_core.preprocess.flat_fielding import flat_fielding
from stp_core.preprocess.dynamic_flatfielding import dff_prepare_plan, dynamic_flat_fielding
from stp_core.preprocess.ring_correction import ring_correction
from stp_core.preprocess.extract_flatdark import extract_flatdark, _medianize
from h5py import File as getHDF5
# pystp-specific:
import stp_core.io.tdf as tdf
def _write_data(lock, im, index, outfile, outshape, outtype, logfilename, cputime, itime):
lock.acquire()
try:
t0 = time()
f_out = getHDF5( outfile, 'a' )
f_out_dset = f_out.require_dataset('exchange/data', outshape, outtype, chunks=tdf.get_dset_chunks(outshape[0]))
tdf.write_sino(f_out_dset,index,im.astype(float32))
# Set minimum and maximum:
if ( amin(im[:]) < float(f_out_dset.attrs['min']) ):
f_out_dset.attrs['min'] = str(amin(im[:]))
if ( amax(im[:]) > float(f_out_dset.attrs['max'])):
f_out_dset.attrs['max'] = str(amax(im[:]))
f_out.close()
t1 = time()
# Print out execution time:
log = open(logfilename,"a")
log.write(linesep + "\tsino_%s processed (CPU: %0.3f sec - I/O: %0.3f sec)." % (str(index).zfill(4), cputime, t1 - t0 + itime))
log.close()
finally:
lock.release()
def _process (lock, int_from, int_to, infile, outfile, outshape, outtype, skipflat, plan, norm_sx, norm_dx, flat_end,
half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ringrem, dynamic_ff, EFF,
filtEFF, im_dark, logfilename):
# Process the required subset of images:
for i in range(int_from, int_to + 1):
# Read input image:
t0 = time()
f_in = getHDF5(infile, 'r')
if "/tomo" in f_in:
dset = f_in['tomo']
else:
dset = f_in['exchange/data']
im = tdf.read_sino(dset,i).astype(float32)
f_in.close()
t1 = time()
# Perform pre-processing (flat fielding, extended FOV, ring removal):
if not skipflat:
if dynamic_ff:
# Dynamic flat fielding with downsampling = 2:
im = dynamic_flat_fielding(im, i, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx)
else:
im = flat_fielding(im, i, plan, flat_end, half_half, half_half_line, norm_sx, norm_dx)
im = extfov_correction(im, ext_fov, ext_fov_rot_right, ext_fov_overlap)
if not skipflat and not dynamic_ff:
im = ring_correction (im, ringrem, flat_end, plan['skip_flat_after'], half_half, half_half_line, ext_fov)
else:
im = ring_correction (im, ringrem, False, False, half_half, half_half_line, ext_fov)
t2 = time()
# Save processed image to HDF5 file (atomic procedure - lock used):
_write_data(lock, im, i, outfile, outshape, outtype, logfilename, t2 - t1, t1 - t0)
def main(argv):
"""To do...
Usage
-----
Parameters
---------
Example
-------
The following line processes the first ten TIFF files of input path
"/home/in" and saves the processed files to "/home/out" with the
application of the Boin and Haibel filter with smoothing via a Butterworth
filter of order 4 and cutoff frequency 0.01:
destripe /home/in /home/out 1 10 1 0.01 4
"""
lock = Lock()
# Get the from and to number of files to process:
int_from = int(argv[0])
int_to = int(argv[1])
# Get paths:
infile = argv[2]
outfile = argv[3]
# Normalization parameters:
norm_sx = int(argv[4])
norm_dx = int(argv[5])
# Params for flat fielding with post flats/darks:
flat_end = True if argv[6] == "True" else False
half_half = True if argv[7] == "True" else False
half_half_line = int(argv[8])
# Params for extended FOV:
ext_fov = True if argv[9] == "True" else False
ext_fov_rot_right = argv[10]
if ext_fov_rot_right == "True":
ext_fov_rot_right = True
if (ext_fov):
norm_sx = 0
else:
ext_fov_rot_right = False
if (ext_fov):
norm_dx = 0
ext_fov_overlap = int(argv[11])
# Method and parameters coded into a string:
ringrem = argv[12]
# Flat fielding method (conventional or dynamic):
dynamic_ff = True if argv[13] == "True" else False
# Nr of threads and log file:
nr_threads = int(argv[14])
logfilename = argv[15]
# Log input parameters:
log = open(logfilename,"w")
log.write(linesep + "\tInput TDF file: %s" % (infile))
log.write(linesep + "\tOutput TDF file: %s" % (outfile))
log.write(linesep + "\t--------------")
log.write(linesep + "\tOpening input dataset...")
log.close()
# Remove a previous copy of output:
if exists(outfile):
remove(outfile)
# Open the HDF5 file:
f_in = getHDF5(infile, 'r')
if "/tomo" in f_in:
dset = f_in['tomo']
tomoprefix = 'tomo'
flatprefix = 'flat'
darkprefix = 'dark'
else:
dset = f_in['exchange/data']
if "/provenance/detector_output" in f_in:
prov_dset = f_in['provenance/detector_output']
tomoprefix = prov_dset.attrs['tomo_prefix']
flatprefix = prov_dset.attrs['flat_prefix']
darkprefix = prov_dset.attrs['dark_prefix']
num_proj = tdf.get_nr_projs(dset)
num_sinos = tdf.get_nr_sinos(dset)
if (num_sinos == 0):
log = open(logfilename,"a")
log.write(linesep + "\tNo projections found. Process will end.")
log.close()
exit()
# Check extrema (int_to == -1 means all files):
if ((int_to >= num_sinos) or (int_to == -1)):
int_to = num_sinos - 1
# Prepare the work plan for flat and dark images:
log = open(logfilename,"a")
log.write(linesep + "\t--------------")
log.write(linesep + "\tPreparing the work plan...")
log.close()
# Extract flat and darks:
skipflat = False
skipdark = False
# Following variables make sense only for dynamic flat fielding:
EFF = -1
filtEFF = -1
im_dark = -1
# Following variable makes sense only for conventional flat fielding:
plan = -1
if not dynamic_ff:
plan = extract_flatdark(f_in, flat_end, logfilename)
if (isscalar(plan['im_flat']) and isscalar(plan['im_flat_after']) ):
skipflat = True
else:
skipflat = False
else:
# Dynamic flat fielding:
if "/tomo" in f_in:
if "/flat" in f_in:
flat_dset = f_in['flat']
if "/dark" in f_in:
im_dark = _medianize(f_in['dark'])
else:
skipdark = True
else:
skipflat = True # Nothing to do in this case
else:
if "/exchange/data_white" in f_in:
flat_dset = f_in['/exchange/data_white']
if "/exchange/data_dark" in f_in:
im_dark = _medianize(f_in['/exchange/data_dark'])
else:
skipdark = True
else:
skipflat = True # Nothing to do in this case
# Prepare plan for dynamic flat fielding with 16 repetitions:
if not skipflat:
EFF, filtEFF = dff_prepare_plan(flat_dset, 16, im_dark)
# Outfile shape can be determined only after first processing in ext FOV mode:
if (ext_fov):
# Read input sino:
idx = num_sinos / 2
im = tdf.read_sino(dset,idx).astype(float32)
im = extfov_correction(im, ext_fov, ext_fov_rot_right, ext_fov_overlap)
# Get the corrected outshape:
outshape = tdf.get_dset_shape(im.shape[1], num_sinos, im.shape[0])
else:
# Get the corrected outshape (in this case it's easy):
im = tdf.read_tomo(dset,0).astype(float32)
outshape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_proj)
# Create the output HDF5 file:
f_out = getHDF5(outfile, 'w')
f_out_dset = f_out.create_dataset('exchange/data', outshape, im.dtype)
f_out_dset.attrs['min'] = str(amin(im[:]))
f_out_dset.attrs['max'] = str(amax(im[:]))
f_out_dset.attrs['version'] = '1.0'
f_out_dset.attrs['axes'] = "y:theta:x"
f_out.close()
f_in.close()
# Log infos:
log = open(logfilename,"a")
log.write(linesep + "\tWork plan prepared correctly.")
log.write(linesep + "\t--------------")
log.write(linesep + "\tPerforming pre processing...")
log.close()
# Run several threads for independent computation without waiting for threads completion:
for num in range(nr_threads):
start = (num_sinos / nr_threads)*num
if (num == nr_threads - 1):
end = num_sinos - 1
else:
end = (num_sinos / nr_threads)*(num + 1) - 1
Process(target=_process, args=(lock, start, end, infile, outfile, outshape, im.dtype, skipflat, plan, norm_sx,
norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ringrem,
dynamic_ff, EFF, filtEFF, im_dark, logfilename )).start()
#start = int_from # 0
#end = int_to # num_sinos - 1
#_process(lock, start, end, infile, outfile, outshape, im.dtype, skipflat, plan, norm_sx,
# norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ringrem,
# dynamic_ff, EFF, filtEFF, im_dark, logfilename)
#255 256 C:\Temp\BrunGeorgos.tdf C:\Temp\BrunGeorgos_corr.tdf 0 0 True True 900 False False 0 rivers:11;0 False 1 C:\Temp\log_00.txt
if __name__ == "__main__":
main(argv[1:])
|