from os import listdir as ls
import png
import pydicom
-from os.path import join
+from os.path import join, dirname
# locals
-from topng import topng
-from regularjson import search, RT_PATH, JSON_GTS, INFA_STR
+from topng import topng, getimgname, mask, CROP_SIZE, EPI_MIN_PIXELS
+from regularjson import search, RT_PATH, INFA_STR, EPI_STR, ENDO_STR
# constants
GLOB_DIR = '../../Data/Images_anonymous/'
-OUT_DIR = './generated/train/'
+OUT_DIR = './generated/updated/'
INDICE_CASE = 'Case'
+PREFIX = '0_' # => 0_Case0002
+
+BLACK_LIST_CASES = [
+ 'Case_0258', 'Case_0259',
+ 'Case_0265', 'Case_0266', 'Case_0300', 'Case_0305',
+ 'Case_0311', 'Case_0316',
+ 'Case_0320', 'Case_0322', 'Case_0325', 'Case_0326', 'Case_0327',
+ 'Case_0328', 'Case_0329', 'Case_0337', 'Case_0338', 'Case_0345',
+ 'Case_0346', 'Case_0348', 'Case_0359', 'Case_0360', 'Case_0361',
+ 'Case_0362', 'Case_0363', 'Case_0364', 'Case_0365', 'Case_0366',
+ 'Case_0367', 'Case_0381', 'Case_0385', 'Case_0400', 'Case_0401', 'Case_0415', 'Case_0416',
+ 'Case_0421','Case_0423', 'Case_0429', 'Case_0442', 'Case_0444', 'Case_0474',
+]
+
+BLACK_LIST_IMAGES = ['Case_0229-Image00002', 'Case_0229-Image00004', 'Case_0229-Image00006', 'Case_0229-Image00007',
+ 'Case_0243-Image00003.png', 'Case_0243-Image00004', 'Case_0243-Image00005', 'Case_0265-Image00001', 'Case_0262-Image00007',
+ 'Case_0262-Image00008', 'Case_0264-Image00006', 'Case_0275-Image00006',
+ 'Case_0307-Image00006', 'Case_0314-Image00001', 'Case_0314-Image00002',
+ 'Case_0314-Image00008', 'Case_0315-Image00005', 'Case_0321-Image00007',
+ 'Case_0321-Image00008', 'Case_0328-Image00002',
+ 'Case_0331-I7000000',
+ 'Case_0356-Image00007', 'Case_0356-Image00008', 'Case_0356-Image00009', 'Case_0356-Image00009',
+ 'Case_0358-Image00008', 'Case_0358-Image00009',
+ 'Case_0377-Image00009',
+]
START = None # to manage the number of Patient Case to use in training + validation, if START == None => 0, if END == None => last index, it will use all in GLOB_DIR
-END = None
-
+END = None # like Case_420, to generate just a part of the dataset
+END_INF_CASES = 309 # to manage correct case with infarctus
+END_NOINF_CASES = None # to manage correct case with noinfarctus
def get(l, i, r):
if len(l) <= 1:
if __name__ == '__main__':
l = sorted(ls(GLOB_DIR))
+ # print("log", l)
+ # exit()
+
# Initiliaze
wmin = hmin = None # None is important here, confert the 'minimum' algo
wmax = hmax = w = h = 0
- for cas in l[START:END]:# cas like 'Case0002'
+ for j, cas in enumerate(l[START:END]):# cas like 'Case0002'
caspath = join(GLOB_DIR, cas)
- if not INDICE_CASE in cas:# if the directory name doesn't sound familiar
+ if not INDICE_CASE in cas: # if the directory name doesn't sound familiar
+ continue
+
+ if cas in BLACK_LIST_CASES: # if the directory was black listed
continue
l1 = sorted( ls( caspath ) )# just ls one CaseXXXX and sort the result
# $$NOTRUSTREF here we need to sort because there is no trusted ref or id
- r = search(RT_PATH, cas)
+ # print('RT_PATH', RT_PATH)
+ # print('PREFIX + cas', PREFIX + cas)
+ r = search(RT_PATH, PREFIX + cas)
+ # print('r', r)
print(cas, end=' ', flush=True) # log CaseXXXX
if not r: # if the feature doesn't yet labeled by an expert go to next
continue
- r = search(r['path'], '.') # to be more dynamic, here can just be '.' instead of '1.2.3.4.5.6'
+
+ # ------<< Searching for .json files
+ r = search(r['path'], 'contours') # contours/1.2.3.4.5.6
+ if not r:
+ continue
+ r = search(r['path'], '1.2.3.4.5.6') # to be more dynamic, here can just be '.' instead of '1.2.3.4.5.6'
+ # ------>>
if r:
r = r['path']
+ # print('ls(r)', ls(r))
try:
l2 = sorted( ls(r) ) # $$NOTRUSTREF
except NotADirectoryError as e:
print("WARN", "a file in unexcepted place")
continue
- if 0 < len(l2) < len(l1):
+ # IF THIS KIND OF ERROR OCCURS, CHANGE ON DISK TO GET IT CONFORMED
+ if 1 == len(l2) < len(l1): # l2 == ["37"] and l1 == ["image001", "image002"..], may be "37" contains searched dirs, so enter..
r = join(r, l2[0])
l2 = sorted( ls(r) ) # $$NOTRUSTREF
# Try once : check subdirectory if i'am not the right
- if 0 < len(l2) < len(l1):
- # TODO: log, json doesn't match with images (labels != features)
- print("WARN", "json doesn't match with images (labels != features), Case", cas)
- continue
- pass
+ # if 0 < len(l2) < len(l1):
+ # # TODO: log, json doesn't match with images (labels != features)
+ # print("WARN", "json doesn't match with images (labels != features), Case", cas)
+ # continue
+ # pass
+
+ for i, dic in enumerate(l1): # l1 => ["image001", "image002"], l2 => ["37", "42"]
+ if True:
+ if "{}-{}".format(cas, dic) in BLACK_LIST_IMAGES: # if image was black listed
+ continue
+
+ epimask = ''
+ infarctus = ''
+ endomask = ''
+ # debug <<
+ # infarctus = True
+ # debug >>
+ # search the right json, for that, we open the json
+ for elt in l2:
+ # print('join', r, l2[i])
+ ref = join(r, elt) # logically, should be the json ref's "parent directory" of i dicom image
+
+ t_epimask = search(ref, EPI_STR)
+
+ parent = dirname(ref) # get the parent dir
+
+ infarctus = search(parent, INFA_STR)
+ endomask = search(parent, ENDO_STR)
+
+ if t_epimask and t_epimask.get("path"):
+ timgname = getimgname(t_epimask["path"])
+
+ if timgname == dic:
+ epimask = t_epimask
+ break
+ else:
+ print('no epicardic')
- for i, dic in enumerate(l1):
- # print('join', r, l2[i])
- ref = join(r, l2[i]) # logically, should be the json ref of i dicom image
- infarctus = search(ref, INFA_STR)
- if infarctus:
+ # print("ref left:", l1[i]) # Testing..
+ # print("ref right:", ref) # Testing..
# print("infarctus:", infarctus) # Testing..
# topng(join(caspath, dic), '%/%-%' % (join(OUT_DIR, 'infarctus'), cas, dic)
- # print(join(caspath, dic), '{}/{}-{}'.format(join(OUT_DIR, 'infarctus'), cas, dic)) # Testing..
- w, h = topng(join(caspath, dic), '{}/{}-{}'.format(join(OUT_DIR, 'infarctus'), cas, dic))
- else:
- # print("no infarctus:", infarctus) # Testing..
- # print(join(caspath, dic), '{}/{}-{}'.format(join(OUT_DIR, 'noinfarctus'), cas, dic)) # Testing..
- w, h = topng(join(caspath, dic), '{}/{}-{}'.format(join(OUT_DIR, 'noinfarctus'), cas, dic))
-
- # search maximums
- if wmax < w: wmax = w
- if hmax < h: hmax = h
-
- # search width minimum
- if wmin is None: wmin = w
- elif wmin > w: wmin = w
-
- # search height minimum
- if hmin is None: hmin = h
- elif hmin > h: hmin = h
-
- print('min-width, max-width:', (wmin, wmax))
- print('min-height, max-height:', (hmin, hmax))
-
- print('Ended!')
+ # print('epimask', epimask)
+
+ # debug <<
+ # epimask = False
+ # debug >>
+
+ if epimask:# this condition could be if necessary
+ if infarctus and (END_INF_CASES is not None) and j > END_INF_CASES:
+ continue
+ if not infarctus and (END_NOINF_CASES is not None) and j > END_NOINF_CASES:
+ continue
+ # topng(join(caspath, dic), '{}/{}-{}'.format(join(OUT_DIR, 'infarctus' if infarctus else 'noinfarctus'), cas, dic), epimask=epimask['path'] if epimask else None)
+ # img_, dicimg_, minmax_ = topng(join(caspath, dic),
+ # '{}/{}-{}'.format(join(OUT_DIR, 'infarctus' if infarctus else 'noinfarctus'), cas, dic),
+ # epimask=epimask['path'] if epimask else None,
+ # verbose=True,
+ # )
+
+ # Possibly data augmentation purpose here
+ print("EPI_MIN_PIXELS", EPI_MIN_PIXELS)
+ topng(
+ join(caspath, dic),
+ outfile='{}/crop-mask/{}-{}'.format(join(OUT_DIR,
+ 'infarctus' if infarctus else 'noinfarctus'),
+ cas,
+ dic,
+ ),
+ epimask=epimask['path'] if epimask else None,
+ centercrop=CROP_SIZE,
+ blackmask=True,
+ epiminpixels=EPI_MIN_PIXELS
+ ) # crop and mask with black
+
+ # if endomask:
+ # topng(
+ # join(caspath, dic),
+ # outfile='{}/crop-mask-hollow/{}-{}'.format(join(OUT_DIR,
+ # 'infarctus' if infarctus else 'noinfarctus'),
+ # cas,
+ # dic,
+ # ),
+ # epimask=epimask['path'] if epimask else None,
+ # endomask=endomask['path'] if endomask else None,
+ # centercrop=CROP_SIZE,
+ # blackmask=True,
+ # ) # crop and (mask and fill hollow) with black
+
+ # topng(
+ # join(caspath, dic),
+ # outfile='{}/crop-nomask/{}-{}'.format(join(OUT_DIR,
+ # 'infarctus' if infarctus else 'noinfarctus'),
+ # cas,
+ # dic,
+ # ),
+ # epimask=epimask['path'] if epimask else None,
+ # centercrop=CROP_SIZE,
+ # ) # just crop, don't apply black mask
+
+ # w, h = BE CAREFULL, be sure w, h is needed, and that topng returns something
+ # topng(
+ # join(caspath, dic),
+ # outfile='{}/contour/{}-{}'.format(join(OUT_DIR,
+ # 'infarctus' if infarctus else 'noinfarctus'),
+ # cas,
+ # dic,
+ # ),
+ # epimask=epimask['path'] if epimask else None,
+ # redcontour=True,
+ # ) # draw a red contour for visibily purpose
+
+ # segmentation width & height <<
+ # xmin, ymin, xmax, ymax = minmax_
+ # w = xmax - xmin + 1 # +1 : even the position takes a bit
+ # h = ymax - ymin + 1
+ # segmentation width & height >>
+
+
+ # # search maximums
+ # if wmax < w: wmax = w
+ # if hmax < h: hmax = h
+
+ # # search width minimum
+ # if wmin is None: wmin = w
+ # elif wmin > w: wmin = w
+
+ # # search height minimum
+ # if hmin is None: hmin = h
+ # elif hmin > h: hmin = h
+ # except Exception as e:
+ # print("WARN", "something went wrong with this image", dic, str(e))
+ # continue
+
+
+ # print('min-width, max-width:', (wmin, wmax))
+ # print('min-height, max-height:', (hmin, hmax))
+
+ # print('Ended!')
