stable, day after meeting

[myo-class.git] / totraindir.py

diff --git a/totraindir.py b/totraindir.py
index b901b85bb7ea34c22bf7730eb83f58f8327e717d..374eb84d102e06bf0b93bb207686799a6f1d1f59 100644 (file)
--- a/totraindir.py
+++ b/totraindir.py
@@ -1,20 +1,46 @@
 from os import listdir as ls
 import png
 import pydicom
 from os import listdir as ls
 import png
 import pydicom

-from os.path import join
+from os.path import join, dirname

 
 # locals
 
 # locals

-from topng import topng, mask, CROP_SIZE
-from regularjson import search, RT_PATH, JSON_GTS, INFA_STR, EPI_STR, ENDO_STR
+from topng import topng, getimgname, mask, CROP_SIZE, EPI_MIN_PIXELS
+from regularjson import search, RT_PATH, INFA_STR, EPI_STR, ENDO_STR

 
 # constants
 GLOB_DIR = '../../Data/Images_anonymous/'
 
 # constants
 GLOB_DIR = '../../Data/Images_anonymous/'

-OUT_DIR = './generated/90/'
+OUT_DIR = './generated/updated/'

 INDICE_CASE = 'Case'
 INDICE_CASE = 'Case'

+PREFIX = '0_' # => 0_Case0002
+
+BLACK_LIST_CASES = [
+       'Case_0258', 'Case_0259',
+       'Case_0265', 'Case_0266', 'Case_0300', 'Case_0305',
+       'Case_0311', 'Case_0316',
+       'Case_0320', 'Case_0322', 'Case_0325', 'Case_0326', 'Case_0327',
+       'Case_0328', 'Case_0329', 'Case_0337', 'Case_0338', 'Case_0345',
+       'Case_0346', 'Case_0348', 'Case_0359', 'Case_0360', 'Case_0361',
+       'Case_0362', 'Case_0363', 'Case_0364', 'Case_0365', 'Case_0366',
+       'Case_0367', 'Case_0381', 'Case_0385', 'Case_0400', 'Case_0401', 'Case_0415', 'Case_0416',
+       'Case_0421','Case_0423', 'Case_0429', 'Case_0442', 'Case_0444', 'Case_0474',
+]
+
+BLACK_LIST_IMAGES = ['Case_0229-Image00002', 'Case_0229-Image00004', 'Case_0229-Image00006', 'Case_0229-Image00007',
+       'Case_0243-Image00003.png', 'Case_0243-Image00004', 'Case_0243-Image00005', 'Case_0265-Image00001', 'Case_0262-Image00007',
+       'Case_0262-Image00008', 'Case_0264-Image00006', 'Case_0275-Image00006',
+       'Case_0307-Image00006', 'Case_0314-Image00001', 'Case_0314-Image00002',
+       'Case_0314-Image00008', 'Case_0315-Image00005', 'Case_0321-Image00007',
+       'Case_0321-Image00008', 'Case_0328-Image00002',
+       'Case_0331-I7000000',
+       'Case_0356-Image00007', 'Case_0356-Image00008', 'Case_0356-Image00009', 'Case_0356-Image00009',
+       'Case_0358-Image00008', 'Case_0358-Image00009',
+       'Case_0377-Image00009', 
+]

 
 START = None # to manage the number of Patient Case to use in training + validation, if START == None => 0, if END == None => last index, it will use all in GLOB_DIR
 
 START = None # to manage the number of Patient Case to use in training + validation, if START == None => 0, if END == None => last index, it will use all in GLOB_DIR

-END = None
-
+END = None # like Case_420, to generate just a part of the dataset
+END_INF_CASES = 309 # to manage correct case with infarctus
+END_NOINF_CASES = None # to manage correct case with noinfarctus

 
 def get(l, i, r):
        if len(l) <= 1:
 
 def get(l, i, r):
        if len(l) <= 1:


@@ -23,134 +49,191 @@ def get(l, i, r):
 if __name__ == '__main__':
        l = sorted(ls(GLOB_DIR))
 
 if __name__ == '__main__':
        l = sorted(ls(GLOB_DIR))
 

+       # print("log", l)
+       # exit()
+

        # Initiliaze 
        wmin = hmin = None # None is important here, confert the 'minimum' algo
        wmax = hmax = w = h = 0
 
        # Initiliaze 
        wmin = hmin = None # None is important here, confert the 'minimum' algo
        wmax = hmax = w = h = 0
 

-       for cas in l[START:END]:# cas like 'Case0002'
+       for j, cas in enumerate(l[START:END]):# cas like 'Case0002'

                caspath = join(GLOB_DIR, cas)
 
                caspath = join(GLOB_DIR, cas)
 

-               if not INDICE_CASE in cas:# if the directory name doesn't sound familiar
+               if not INDICE_CASE in cas: # if the directory name doesn't sound familiar
+                       continue
+
+               if cas in BLACK_LIST_CASES: # if the directory was black listed

                        continue
 
                l1 = sorted( ls( caspath ) )# just ls one CaseXXXX and sort the result
                # $$NOTRUSTREF here we need to sort because there is no trusted ref or id
 
                        continue
 
                l1 = sorted( ls( caspath ) )# just ls one CaseXXXX and sort the result
                # $$NOTRUSTREF here we need to sort because there is no trusted ref or id
 

-               r = search(RT_PATH, cas)
+               # print('RT_PATH', RT_PATH)
+               # print('PREFIX + cas', PREFIX + cas)
+               r = search(RT_PATH, PREFIX + cas)
+               # print('r', r)

 
                print(cas, end=' ', flush=True) # log CaseXXXX
 
                if not r: # if the feature doesn't yet labeled by an expert go to next
                        continue
 
                print(cas, end=' ', flush=True) # log CaseXXXX
 
                if not r: # if the feature doesn't yet labeled by an expert go to next
                        continue

-               r = search(r['path'], '.') # to be more dynamic, here can just be '.' instead of '1.2.3.4.5.6'
+
+               # ------<< Searching for .json files
+               r = search(r['path'], 'contours') # contours/1.2.3.4.5.6
+               if not r:
+                       continue
+               r = search(r['path'], '1.2.3.4.5.6') # to be more dynamic, here can just be '.' instead of '1.2.3.4.5.6'
+               # ------>>

 
                if r:
                        r = r['path']
 
                if r:
                        r = r['path']

+                       # print('ls(r)', ls(r))

                        try:
                                l2 = sorted( ls(r) ) # $$NOTRUSTREF
                        except NotADirectoryError as e:
                                print("WARN", "a file in unexcepted place")
                                continue
 
                        try:
                                l2 = sorted( ls(r) ) # $$NOTRUSTREF
                        except NotADirectoryError as e:
                                print("WARN", "a file in unexcepted place")
                                continue
 

-                       if 0 < len(l2) < len(l1):
+                       # IF THIS KIND OF ERROR OCCURS, CHANGE ON DISK TO GET IT CONFORMED
+                       if 1 == len(l2) < len(l1): # l2 == ["37"] and l1 == ["image001", "image002"..], may be "37" contains searched dirs, so enter..

                                r = join(r, l2[0])
                                l2 = sorted( ls(r) ) # $$NOTRUSTREF
                                # Try once : check subdirectory if i'am not the right
 
                                r = join(r, l2[0])
                                l2 = sorted( ls(r) ) # $$NOTRUSTREF
                                # Try once : check subdirectory if i'am not the right
 

-                       if 0 < len(l2) < len(l1):
-                               # TODO: log, json doesn't match with images (labels != features)
-                               print("WARN", "json doesn't match with images (labels != features), Case", cas)
-                               continue
-                               pass
-
-                       for i, dic in enumerate(l1):
-                               # print('join', r, l2[i])
-                               ref = join(r, l2[i]) # logically, should be the json ref's "parent directory" of i dicom image
-
-                               infarctus = search(ref, INFA_STR)
-                               epimask = search(ref, EPI_STR)
-                               endomask = search(ref, ENDO_STR)
-                               # print("ref left:", l1[i]) # Testing..
-                               # print("ref right:", ref) # Testing..
-                               # print("infarctus:", infarctus) # Testing..
-                               # topng(join(caspath, dic), '%/%-%' % (join(OUT_DIR, 'infarctus'), cas, dic)
-                               # print('epimask', epimask)
-
-                               if epimask:# this condition could be if necessary
-                                       # w, h = topng(join(caspath, dic), '{}/{}-{}'.format(join(OUT_DIR, 'infarctus' if infarctus else 'noinfarctus'), cas, dic), epimask=epimask['path'] if epimask else None)
-                                       # img_, dicimg_, minmax_ = topng(join(caspath, dic),
-                                       #       '{}/{}-{}'.format(join(OUT_DIR, 'infarctus' if infarctus else 'noinfarctus'), cas, dic),
-                                       #       epimask=epimask['path'] if epimask else None,
-                                       #       verbose=True,
-                                       # )
-
-                                       # Possibly data augmentation purpose here
-                                       w, h = topng(
-                                               join(caspath, dic),
-                                               outfile='{}/crop-mask/{}-{}'.format(join(OUT_DIR,
-                                                       'infarctus' if infarctus else 'noinfarctus'),
-                                                       cas,
-                                                       dic,
-                                               ),
-                                               epimask=epimask['path'] if epimask else None,
-                                               centercrop=CROP_SIZE,
-                                               blackmask=True,
-                                       ) # crop and mask with black
-
-                                       if endomask:
-                                               w, h = topng(
+                       # if 0 < len(l2) < len(l1):
+                       #       # TODO: log, json doesn't match with images (labels != features)
+                       #       print("WARN", "json doesn't match with images (labels != features), Case", cas)
+                       #       continue
+                       #       pass
+
+                       for i, dic in enumerate(l1): # l1 => ["image001", "image002"], l2 => ["37", "42"]
+                               if True:
+                                       if "{}-{}".format(cas, dic) in BLACK_LIST_IMAGES: # if image was black listed
+                                               continue
+
+                                       epimask = ''
+                                       infarctus = ''
+                                       endomask = ''
+                                       # debug <<
+                                       # infarctus = True
+                                       # debug >>
+                                       # search the right json, for that, we open the json
+                                       for elt in l2:
+                                               # print('join', r, l2[i])
+                                               ref = join(r, elt) # logically, should be the json ref's "parent directory" of i dicom image
+
+                                               t_epimask = search(ref, EPI_STR)
+
+                                               parent = dirname(ref) # get the parent dir
+
+                                               infarctus = search(parent, INFA_STR)
+                                               endomask = search(parent, ENDO_STR)
+
+                                               if t_epimask and t_epimask.get("path"):
+                                                       timgname = getimgname(t_epimask["path"])
+
+                                                       if timgname == dic:
+                                                               epimask = t_epimask
+                                                               break
+                                               else:
+                                                       print('no epicardic')
+
+
+                                       # print("ref left:", l1[i]) # Testing..
+                                       # print("ref right:", ref) # Testing..
+                                       # print("infarctus:", infarctus) # Testing..
+                                       # topng(join(caspath, dic), '%/%-%' % (join(OUT_DIR, 'infarctus'), cas, dic)
+                                       # print('epimask', epimask)
+
+                                       # debug <<
+                                       # epimask = False
+                                       # debug >>
+
+                                       if epimask:# this condition could be if necessary
+                                               if infarctus and (END_INF_CASES is not None) and j > END_INF_CASES:
+                                                       continue
+                                               if not infarctus and (END_NOINF_CASES is not None) and j > END_NOINF_CASES:
+                                                       continue
+                                               # topng(join(caspath, dic), '{}/{}-{}'.format(join(OUT_DIR, 'infarctus' if infarctus else 'noinfarctus'), cas, dic), epimask=epimask['path'] if epimask else None)
+                                               # img_, dicimg_, minmax_ = topng(join(caspath, dic),
+                                               #       '{}/{}-{}'.format(join(OUT_DIR, 'infarctus' if infarctus else 'noinfarctus'), cas, dic),
+                                               #       epimask=epimask['path'] if epimask else None,
+                                               #       verbose=True,
+                                               # )
+
+                                               # Possibly data augmentation purpose here
+                                               print("EPI_MIN_PIXELS", EPI_MIN_PIXELS)
+                                               topng(

                                                        join(caspath, dic),
                                                        join(caspath, dic),

-                                                       outfile='{}/crop-mask-hollow/{}-{}'.format(join(OUT_DIR,
+                                                       outfile='{}/crop-mask/{}-{}'.format(join(OUT_DIR,

                                                                'infarctus' if infarctus else 'noinfarctus'),
                                                                cas,
                                                                dic,
                                                        ),
                                                        epimask=epimask['path'] if epimask else None,
                                                                'infarctus' if infarctus else 'noinfarctus'),
                                                                cas,
                                                                dic,
                                                        ),
                                                        epimask=epimask['path'] if epimask else None,

-                                                       endomask=endomask['path'] if endomask else None,

                                                        centercrop=CROP_SIZE,
                                                        blackmask=True,
                                                        centercrop=CROP_SIZE,
                                                        blackmask=True,

-                                               ) # crop and (mask and fill hollow) with black
-
-                                       w, h = topng(
-                                               join(caspath, dic),
-                                               outfile='{}/crop-nomask/{}-{}'.format(join(OUT_DIR,
-                                                       'infarctus' if infarctus else 'noinfarctus'),
-                                                       cas,
-                                                       dic,
-                                               ),
-                                               epimask=epimask['path'] if epimask else None,
-                                               centercrop=CROP_SIZE,
-                                       ) # just crop, don't apply black mask
-
-                                       # w, h = topng(
-                                       #       join(caspath, dic),
-                                       #       outfile='{}/contour/{}-{}'.format(join(OUT_DIR,
-                                       #               'infarctus' if infarctus else 'noinfarctus'),
-                                       #               cas,
-                                       #               dic,
-                                       #       ),
-                                       #       epimask=epimask['path'] if epimask else None,
-                                       #       redcontour=True,
-                                       # ) # draw a red contour for visibily purpose
-
-                                       # segmentation width & height <<
-                                       # xmin, ymin, xmax, ymax = minmax_
-                                       # w = xmax - xmin + 1 # +1 : even the position takes a bit
-                                       # h = ymax - ymin + 1
-                                       # segmentation width & height >>
-
-
-                                       # # search maximums
-                                       # if wmax < w: wmax = w 
-                                       # if hmax < h: hmax = h
-
-                                       # # search width minimum
-                                       # if wmin is None: wmin = w
-                                       # elif wmin > w: wmin = w
-                                       
-                                       # # search height minimum
-                                       # if hmin is None: hmin = h
-                                       # elif hmin > h: hmin = h
+                                                       epiminpixels=EPI_MIN_PIXELS
+                                               ) # crop and mask with black
+
+                                               # if endomask:
+                                               # topng(
+                                               #               join(caspath, dic),
+                                               #               outfile='{}/crop-mask-hollow/{}-{}'.format(join(OUT_DIR,
+                                               #                       'infarctus' if infarctus else 'noinfarctus'),
+                                               #                       cas,
+                                               #                       dic,
+                                               #               ),
+                                               #               epimask=epimask['path'] if epimask else None,
+                                               #               endomask=endomask['path'] if endomask else None,
+                                               #               centercrop=CROP_SIZE,
+                                               #               blackmask=True,
+                                               #       ) # crop and (mask and fill hollow) with black
+
+                                               # topng(
+                                               #       join(caspath, dic),
+                                               #       outfile='{}/crop-nomask/{}-{}'.format(join(OUT_DIR,
+                                               #               'infarctus' if infarctus else 'noinfarctus'),
+                                               #               cas,
+                                               #               dic,
+                                               #       ),
+                                               #       epimask=epimask['path'] if epimask else None,
+                                               #       centercrop=CROP_SIZE,
+                                               # ) # just crop, don't apply black mask
+
+                                               # w, h = BE CAREFULL, be sure w, h is needed, and that topng returns something
+                                               # topng(
+                                               #       join(caspath, dic),
+                                               #       outfile='{}/contour/{}-{}'.format(join(OUT_DIR,
+                                               #               'infarctus' if infarctus else 'noinfarctus'),
+                                               #               cas,
+                                               #               dic,
+                                               #       ),
+                                               #       epimask=epimask['path'] if epimask else None,
+                                               #       redcontour=True,
+                                               # ) # draw a red contour for visibily purpose
+
+                                               # segmentation width & height <<
+                                               # xmin, ymin, xmax, ymax = minmax_
+                                               # w = xmax - xmin + 1 # +1 : even the position takes a bit
+                                               # h = ymax - ymin + 1
+                                               # segmentation width & height >>
+
+
+                                               # # search maximums
+                                               # if wmax < w: wmax = w 
+                                               # if hmax < h: hmax = h
+
+                                               # # search width minimum
+                                               # if wmin is None: wmin = w
+                                               # elif wmin > w: wmin = w
+                                               
+                                               # # search height minimum
+                                               # if hmin is None: hmin = h
+                                               # elif hmin > h: hmin = h
+                               # except Exception as e:
+                               #       print("WARN", "something went wrong with this image", dic, str(e))
+                               #       continue

 
 
        # print('min-width, max-width:', (wmin, wmax))
 
 
        # print('min-width, max-width:', (wmin, wmax))