Refactoring, fin du lever/coucher de soleil, et début de sentinelles

[predictops.git] / predictops / engine.py

diff --git a/predictops/engine.py b/predictops/engine.py
index e7bbf1c5aa58221da7a8aaa71788cf0339258cbc..bb85787ca304aa2643a5c1e2c42e53362ced5689 100644 (file)
--- a/predictops/engine.py
+++ b/predictops/engine.py
@@ -5,97 +5,154 @@ from logging.config import fileConfig
 from pathlib import Path
 from shutil import rmtree
 
 from pathlib import Path
 from shutil import rmtree
 

+import os
+import pytz
+
+from .learn.learning import Learning
+from .learn.preprocessing import Preprocessing

 from .source.ephemeris import Ephemeris
 from .source.holidays import Holidays
 from .source.meteofrance import MeteoFrance
 from .source.ephemeris import Ephemeris
 from .source.holidays import Holidays
 from .source.meteofrance import MeteoFrance

-from .learn.learning import Learning
-from .learn.preprocessing import Preprocessing
+from .source.ramadan import Ramadan
+from .source.sentinelles import Sentinelles

 from .target.target import Target
 
 fileConfig((Path.cwd() / 'config') / 'logging.cfg')
 logger = getLogger()
 
 
 from .target.target import Target
 
 fileConfig((Path.cwd() / 'config') / 'logging.cfg')
 logger = getLogger()
 
 

-class Engine:
+class Engine(object):

 
 

-    def __init__(self, config_file = (Path.cwd() / 'config') / 'learn.cfg'):
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        with open(str(self._file_name / os.path.basename(self._file_name)) + '.cfg', 'w') as f:
+            f.write(self._config_text)
+
+    def __init__(self, config_file=(Path.cwd() / 'config') / 'learn.cfg'):

         self._config = ConfigParser()
         self._config.read(config_file)
         self._config = ConfigParser()
         self._config.read(config_file)

+        launching_time = datetime.strftime(datetime.now(), '%Y_%m_%d_%H_%M')
+        self._name = os.path.splitext(os.path.basename(eval(self._config['TARGET']['config'])))[0]
+        self._file_name = f"{self._name}-{launching_time}"
+        p = Path.cwd() / 'results' / self._name
+        p.mkdir(exist_ok=True, parents=True)
+        self._file_name = p / self._file_name
+
+        self._config_text = ''
+        with open(config_file) as f:
+            self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
+            self._config_text += f.read() + '\n\n'
+

         self._start = datetime.strptime(self._config['DATETIME']['start'],
                                         '%m/%d/%Y %H:%M:%S')
         self._end = datetime.strptime(self._config['DATETIME']['end'],
         self._start = datetime.strptime(self._config['DATETIME']['start'],
                                         '%m/%d/%Y %H:%M:%S')
         self._end = datetime.strptime(self._config['DATETIME']['end'],

-                                        '%m/%d/%Y %H:%M:%S')
+                                      '%m/%d/%Y %H:%M:%S')

 
 

-        self._timestep = timedelta(hours =
-                                   self._config['DATETIME'].getfloat('hourStep'))
+        self._timestep = timedelta(hours=self._config['DATETIME'].getfloat('hourStep'))

 
         self._X = {}
 
 
         self._X = {}
 

-
-

     def clean(self):
         # Cleaning the data directory
         logger.info("Cleaning and restoring data directory")
     def clean(self):
         # Cleaning the data directory
         logger.info("Cleaning and restoring data directory")

-        directory  = Path.cwd() / 'data'
+        directory = Path.cwd() / 'data'

         if directory.is_dir():
             rmtree(directory)
         p = Path(Path.cwd() / 'data')
         p.mkdir()
 
         if directory.is_dir():
             rmtree(directory)
         p = Path(Path.cwd() / 'data')
         p.mkdir()
 

-

     def add_features(self):
     def add_features(self):

-        if self._config['FEATURES'].getboolean('meteofrance'):
-            meteofeature = MeteoFrance(config_file =
-                                       eval(self._config['FEATURE_CONFIG']['meteofrance']))
-
-            meteofeature.start = self._start
-            meteofeature.end = self._end
-
-            meteofeature.update()
-            dated_features = meteofeature.dated_features
-            for date in dated_features:
-                self._X.setdefault(date,{}).update(dated_features[date])
-

         if self._config['FEATURES'].getboolean('ephemeris'):
         if self._config['FEATURES'].getboolean('ephemeris'):

-            ephemerides = Ephemeris(config_file =
-                                    eval(self._config['FEATURE_CONFIG']['ephemeris']))
+            config_file = eval(self._config['FEATURE_CONFIG']['ephemeris'])
+            with open(config_file) as f:
+                self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
+                self._config_text += f.read() + '\n\n'
+
+            ephemerides = Ephemeris(config_file=config_file, start=self._start, end=self._end)

 
 

-            ephemerides.start = self._start
-            ephemerides.end = self._end
+            # ephemerides.start = self._start
+            # ephemerides.end = self._end

 
             dated_features = ephemerides.dated_features
             for date in dated_features:
 
             dated_features = ephemerides.dated_features
             for date in dated_features:

-                self._X.setdefault(date,{}).update(dated_features[date])
+                self._X.setdefault(date, {}).update(dated_features[date])

 
         if self._config['FEATURES'].getboolean('holidays'):
 
         if self._config['FEATURES'].getboolean('holidays'):

-            holidays = Holidays(config_file =
-                                eval(self._config['FEATURE_CONFIG']['holidays']))
+            config_file = eval(self._config['FEATURE_CONFIG']['holidays'])
+            with open(config_file) as f:
+                self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
+                self._config_text += f.read() + '\n\n'
+
+            holidays = Holidays(config_file=config_file)

 
             holidays.start = self._start
             holidays.end = self._end
 
             dated_features = holidays.dated_features
             for date in dated_features:
 
             holidays.start = self._start
             holidays.end = self._end
 
             dated_features = holidays.dated_features
             for date in dated_features:

-                self._X.setdefault(date,{}).update(dated_features[date])
+                self._X.setdefault(date, {}).update(dated_features[date])
+
+        if self._config['FEATURES'].getboolean('meteofrance'):
+            config_file = eval(self._config['FEATURE_CONFIG']['meteofrance'])
+            with open(config_file) as f:
+                self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
+                self._config_text += f.read() + '\n\n'

 
 

+            meteofeature = MeteoFrance(config_file=config_file)
+
+            meteofeature.start = self._start
+            meteofeature.end = self._end
+
+            meteofeature.update()
+            dated_features = meteofeature.dated_features
+            for date in dated_features:
+                self._X.setdefault(date, {}).update(dated_features[date])
+
+        if self._config['FEATURES'].getboolean('ramadan'):
+            config_file = eval(self._config['FEATURE_CONFIG']['ramadan'])
+            with open(config_file) as f:
+                self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
+                self._config_text += f.read() + '\n\n'
+
+            ramadan = Ramadan(config_file=config_file)
+
+            ramadan.start = self._start
+            ramadan.end = self._end
+
+            dated_features = ramadan.dated_features
+            for date in dated_features:
+                self._X.setdefault(date, {}).update(dated_features[date])

 
     def add_target(self):
 
     def add_target(self):

-        self._target = Target(config_file = eval(self._config['TARGET']['config']),
-                              start = self._start, end = self._end,
-                              timestep = self._timestep)
+        config_file = eval(self._config['TARGET']['config'])
+        cumulative = self._config['TARGET'].getboolean('cumulative')
+        with open(config_file) as f:
+            self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
+            self._config_text += f.read() + '\n\n'

 
 

+        self._target = Target(config_file=config_file,
+                              start=self._start, end=self._end,
+                              timestep=self._timestep, cumulative=cumulative)

 
     def add_preprocessing(self):
 
     def add_preprocessing(self):

-        self._preproc = Preprocessing(config_file = self._config,
-                                      dict_features = self.X,
-                                      dict_target = self.y)
-
+        self._preproc = Preprocessing(config_file=self._config,
+                                      start=self._start, end=self._end,
+                                      timestep=self._timestep,
+                                      dict_features=self.X,
+                                      dict_target=self.y)

 
     def learn(self):
 
     def learn(self):

-        history = self._config['HISTORY_KNOWLEDGE'].getint('nb_lines')
-        self._learner = Learning(config_file = eval(self._config['LEARNER']['config']),
-                                 X = self._preproc.dataframe, y = list(self.y.values())[history:])
+        config_file = eval(self._config['LEARNER']['config'])
+        with open(config_file) as f:
+            self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
+            self._config_text += f.read() + '\n\n'

 
 

+        history = eval(self._config['HISTORY_KNOWLEDGE']['nb_lines'])
+        self._learner = Learning(config_file=config_file, file_name=self._file_name,
+                                 X=self._preproc.dataframe, y=list(self.y.values())[history:],
+                                 horizon=self._config['TARGET'].getint('horizon'))

 
     @property
     def X(self):
 
     @property
     def X(self):


@@ -105,7 +162,6 @@ class Engine:
     def X(self, x):
         self._X = x
 
     def X(self, x):
         self._X = x
 

-

     @property
     def y(self):
         return self._target.y
     @property
     def y(self):
         return self._target.y