totraindir.py

   1 from os import listdir as ls
   2 import png
   3 import pydicom
   4 from os.path import join, dirname
   5
   6 # locals
   7 from topng import topng, getimgname, mask, CROP_SIZE, EPI_MIN_PIXELS
   8 from regularjson import search, RT_PATH, INFA_STR, EPI_STR, ENDO_STR
   9
  10 # constants
  11 GLOB_DIR = '../../Data/Images_anonymous/'
  12 OUT_DIR = './generated/updated/'
  13 INDICE_CASE = 'Case'
  14 PREFIX = '0_' # => 0_Case0002
  15
  16 BLACK_LIST_CASES = [
  17         'Case_0258', 'Case_0259',
  18         'Case_0265', 'Case_0266', 'Case_0300', 'Case_0305',
  19         'Case_0311', 'Case_0316',
  20         'Case_0320', 'Case_0322', 'Case_0325', 'Case_0326', 'Case_0327',
  21         'Case_0328', 'Case_0329', 'Case_0337', 'Case_0338', 'Case_0345',
  22         'Case_0346', 'Case_0348', 'Case_0359', 'Case_0360', 'Case_0361',
  23         'Case_0362', 'Case_0363', 'Case_0364', 'Case_0365', 'Case_0366',
  24         'Case_0367', 'Case_0381', 'Case_0385', 'Case_0400', 'Case_0401', 'Case_0415', 'Case_0416',
  25         'Case_0421','Case_0423', 'Case_0429', 'Case_0442', 'Case_0444', 'Case_0474',
  26 ]
  27
  28 BLACK_LIST_IMAGES = ['Case_0229-Image00002', 'Case_0229-Image00004', 'Case_0229-Image00006', 'Case_0229-Image00007',
  29         'Case_0243-Image00003.png', 'Case_0243-Image00004', 'Case_0243-Image00005', 'Case_0265-Image00001', 'Case_0262-Image00007',
  30         'Case_0262-Image00008', 'Case_0264-Image00006', 'Case_0275-Image00006',
  31         'Case_0307-Image00006', 'Case_0314-Image00001', 'Case_0314-Image00002',
  32         'Case_0314-Image00008', 'Case_0315-Image00005', 'Case_0321-Image00007',
  33         'Case_0321-Image00008', 'Case_0328-Image00002',
  34         'Case_0331-I7000000',
  35         'Case_0356-Image00007', 'Case_0356-Image00008', 'Case_0356-Image00009', 'Case_0356-Image00009',
  36         'Case_0358-Image00008', 'Case_0358-Image00009',
  37         'Case_0377-Image00009',
  38 ]
  39
  40 START = None # to manage the number of Patient Case to use in training + validation, if START == None => 0, if END == None => last index, it will use all in GLOB_DIR
  41 END = None # like Case_420, to generate just a part of the dataset
  42 END_INF_CASES = 309 # to manage correct case with infarctus
  43 END_NOINF_CASES = None # to manage correct case with noinfarctus
  44
  45 def get(l, i, r):
  46         if len(l) <= 1:
  47                 return sorted( ls(r) )
  48
  49 if __name__ == '__main__':
  50         l = sorted(ls(GLOB_DIR))
  51
  52         # print("log", l)
  53         # exit()
  54
  55         # Initiliaze
  56         wmin = hmin = None # None is important here, confert the 'minimum' algo
  57         wmax = hmax = w = h = 0
  58
  59         for j, cas in enumerate(l[START:END]):# cas like 'Case0002'
  60                 caspath = join(GLOB_DIR, cas)
  61
  62                 if not INDICE_CASE in cas: # if the directory name doesn't sound familiar
  63                         continue
  64
  65                 if cas in BLACK_LIST_CASES: # if the directory was black listed
  66                         continue
  67
  68                 l1 = sorted( ls( caspath ) )# just ls one CaseXXXX and sort the result
  69                 # $$NOTRUSTREF here we need to sort because there is no trusted ref or id
  70
  71                 # print('RT_PATH', RT_PATH)
  72                 # print('PREFIX + cas', PREFIX + cas)
  73                 r = search(RT_PATH, PREFIX + cas)
  74                 # print('r', r)
  75
  76                 print(cas, end=' ', flush=True) # log CaseXXXX
  77
  78                 if not r: # if the feature doesn't yet labeled by an expert go to next
  79                         continue
  80
  81                 # ------<< Searching for .json files
  82                 r = search(r['path'], 'contours') # contours/1.2.3.4.5.6
  83                 if not r:
  84                         continue
  85                 r = search(r['path'], '1.2.3.4.5.6') # to be more dynamic, here can just be '.' instead of '1.2.3.4.5.6'
  86                 # ------>>
  87
  88                 if r:
  89                         r = r['path']
  90                         # print('ls(r)', ls(r))
  91                         try:
  92                                 l2 = sorted( ls(r) ) # $$NOTRUSTREF
  93                         except NotADirectoryError as e:
  94                                 print("WARN", "a file in unexcepted place")
  95                                 continue
  96
  97                         # IF THIS KIND OF ERROR OCCURS, CHANGE ON DISK TO GET IT CONFORMED
  98                         if 1 == len(l2) < len(l1): # l2 == ["37"] and l1 == ["image001", "image002"..], may be "37" contains searched dirs, so enter..
  99                                 r = join(r, l2[0])
 100                                 l2 = sorted( ls(r) ) # $$NOTRUSTREF
 101                                 # Try once : check subdirectory if i'am not the right
 102
 103                         # if 0 < len(l2) < len(l1):
 104                         #       # TODO: log, json doesn't match with images (labels != features)
 105                         #       print("WARN", "json doesn't match with images (labels != features), Case", cas)
 106                         #       continue
 107                         #       pass
 108
 109                         for i, dic in enumerate(l1): # l1 => ["image001", "image002"], l2 => ["37", "42"]
 110                                 if True:
 111                                         if "{}-{}".format(cas, dic) in BLACK_LIST_IMAGES: # if image was black listed
 112                                                 continue
 113
 114                                         epimask = ''
 115                                         infarctus = ''
 116                                         endomask = ''
 117                                         # debug <<
 118                                         # infarctus = True
 119                                         # debug >>
 120                                         # search the right json, for that, we open the json
 121                                         for elt in l2:
 122                                                 # print('join', r, l2[i])
 123                                                 ref = join(r, elt) # logically, should be the json ref's "parent directory" of i dicom image
 124
 125                                                 t_epimask = search(ref, EPI_STR)
 126
 127                                                 parent = dirname(ref) # get the parent dir
 128
 129                                                 infarctus = search(parent, INFA_STR)
 130                                                 endomask = search(parent, ENDO_STR)
 131
 132                                                 if t_epimask and t_epimask.get("path"):
 133                                                         timgname = getimgname(t_epimask["path"])
 134
 135                                                         if timgname == dic:
 136                                                                 epimask = t_epimask
 137                                                                 break
 138                                                 else:
 139                                                         print('no epicardic')
 140
 141
 142                                         # print("ref left:", l1[i]) # Testing..
 143                                         # print("ref right:", ref) # Testing..
 144                                         # print("infarctus:", infarctus) # Testing..
 145                                         # topng(join(caspath, dic), '%/%-%' % (join(OUT_DIR, 'infarctus'), cas, dic)
 146                                         # print('epimask', epimask)
 147
 148                                         # debug <<
 149                                         # epimask = False
 150                                         # debug >>
 151
 152                                         if epimask:# this condition could be if necessary
 153                                                 if infarctus and (END_INF_CASES is not None) and j > END_INF_CASES:
 154                                                         continue
 155                                                 if not infarctus and (END_NOINF_CASES is not None) and j > END_NOINF_CASES:
 156                                                         continue
 157                                                 # topng(join(caspath, dic), '{}/{}-{}'.format(join(OUT_DIR, 'infarctus' if infarctus else 'noinfarctus'), cas, dic), epimask=epimask['path'] if epimask else None)
 158                                                 # img_, dicimg_, minmax_ = topng(join(caspath, dic),
 159                                                 #       '{}/{}-{}'.format(join(OUT_DIR, 'infarctus' if infarctus else 'noinfarctus'), cas, dic),
 160                                                 #       epimask=epimask['path'] if epimask else None,
 161                                                 #       verbose=True,
 162                                                 # )
 163
 164                                                 # Possibly data augmentation purpose here
 165                                                 print("EPI_MIN_PIXELS", EPI_MIN_PIXELS)
 166                                                 topng(
 167                                                         join(caspath, dic),
 168                                                         outfile='{}/crop-mask/{}-{}'.format(join(OUT_DIR,
 169                                                                 'infarctus' if infarctus else 'noinfarctus'),
 170                                                                 cas,
 171                                                                 dic,
 172                                                         ),
 173                                                         epimask=epimask['path'] if epimask else None,
 174                                                         centercrop=CROP_SIZE,
 175                                                         blackmask=True,
 176                                                         epiminpixels=EPI_MIN_PIXELS
 177                                                 ) # crop and mask with black
 178
 179                                                 # if endomask:
 180                                                 # topng(
 181                                                 #               join(caspath, dic),
 182                                                 #               outfile='{}/crop-mask-hollow/{}-{}'.format(join(OUT_DIR,
 183                                                 #                       'infarctus' if infarctus else 'noinfarctus'),
 184                                                 #                       cas,
 185                                                 #                       dic,
 186                                                 #               ),
 187                                                 #               epimask=epimask['path'] if epimask else None,
 188                                                 #               endomask=endomask['path'] if endomask else None,
 189                                                 #               centercrop=CROP_SIZE,
 190                                                 #               blackmask=True,
 191                                                 #       ) # crop and (mask and fill hollow) with black
 192
 193                                                 # topng(
 194                                                 #       join(caspath, dic),
 195                                                 #       outfile='{}/crop-nomask/{}-{}'.format(join(OUT_DIR,
 196                                                 #               'infarctus' if infarctus else 'noinfarctus'),
 197                                                 #               cas,
 198                                                 #               dic,
 199                                                 #       ),
 200                                                 #       epimask=epimask['path'] if epimask else None,
 201                                                 #       centercrop=CROP_SIZE,
 202                                                 # ) # just crop, don't apply black mask
 203
 204                                                 # w, h = BE CAREFULL, be sure w, h is needed, and that topng returns something
 205                                                 # topng(
 206                                                 #       join(caspath, dic),
 207                                                 #       outfile='{}/contour/{}-{}'.format(join(OUT_DIR,
 208                                                 #               'infarctus' if infarctus else 'noinfarctus'),
 209                                                 #               cas,
 210                                                 #               dic,
 211                                                 #       ),
 212                                                 #       epimask=epimask['path'] if epimask else None,
 213                                                 #       redcontour=True,
 214                                                 # ) # draw a red contour for visibily purpose
 215
 216                                                 # segmentation width & height <<
 217                                                 # xmin, ymin, xmax, ymax = minmax_
 218                                                 # w = xmax - xmin + 1 # +1 : even the position takes a bit
 219                                                 # h = ymax - ymin + 1
 220                                                 # segmentation width & height >>
 221
 222
 223                                                 # # search maximums
 224                                                 # if wmax < w: wmax = w
 225                                                 # if hmax < h: hmax = h
 226
 227                                                 # # search width minimum
 228                                                 # if wmin is None: wmin = w
 229                                                 # elif wmin > w: wmin = w
 230
 231                                                 # # search height minimum
 232                                                 # if hmin is None: hmin = h
 233                                                 # elif hmin > h: hmin = h
 234                                 # except Exception as e:
 235                                 #       print("WARN", "something went wrong with this image", dic, str(e))
 236                                 #       continue
 237
 238
 239         # print('min-width, max-width:', (wmin, wmax))
 240         # print('min-height, max-height:', (hmin, hmax))
 241
 242         # print('Ended!')