predictops/engine.py

   1 from configparser import ConfigParser
   2 from datetime import datetime, timedelta
   3 from logging import getLogger
   4 from logging.config import fileConfig
   5 from pathlib import Path
   6 from shutil import rmtree
   7
   8 import os
   9 import pytz
  10
  11 from .learn.learning import Learning
  12 from .learn.preprocessing import Preprocessing
  13 from .source.ephemeris import Ephemeris
  14 from .source.holidays import Holidays
  15 from .source.meteofrance import MeteoFrance
  16 from .source.ramadan import Ramadan
  17 from .source.sentinelles import Sentinelles
  18 from .target.target import Target
  19
  20 fileConfig((Path.cwd() / 'config') / 'logging.cfg')
  21 logger = getLogger()
  22
  23
  24 class Engine(object):
  25
  26     def __enter__(self):
  27         return self
  28
  29     def __exit__(self, type, value, traceback):
  30         with open(str(self._file_name / os.path.basename(self._file_name)) + '.cfg', 'w') as f:
  31             f.write(self._config_text)
  32
  33     def __init__(self, config_file=(Path.cwd() / 'config') / 'learn.cfg'):
  34         self._config = ConfigParser()
  35         self._config.read(config_file)
  36         launching_time = datetime.strftime(datetime.now(), '%Y_%m_%d_%H_%M')
  37         self._name = os.path.splitext(os.path.basename(eval(self._config['TARGET']['config'])))[0]
  38         self._file_name = f"{self._name}-{launching_time}"
  39         p = Path.cwd() / 'results' / self._name
  40         p.mkdir(exist_ok=True, parents=True)
  41         self._file_name = p / self._file_name
  42
  43         self._config_text = ''
  44         with open(config_file) as f:
  45             self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
  46             self._config_text += f.read() + '\n\n'
  47
  48         self._start = datetime.strptime(self._config['DATETIME']['start'],
  49                                         '%m/%d/%Y %H:%M:%S')
  50         self._end = datetime.strptime(self._config['DATETIME']['end'],
  51                                       '%m/%d/%Y %H:%M:%S')
  52
  53         self._timestep = timedelta(hours=self._config['DATETIME'].getfloat('hourStep'))
  54
  55         self._X = {}
  56
  57     def clean(self):
  58         # Cleaning the data directory
  59         logger.info("Cleaning and restoring data directory")
  60         directory = Path.cwd() / 'data'
  61         if directory.is_dir():
  62             rmtree(directory)
  63         p = Path(Path.cwd() / 'data')
  64         p.mkdir()
  65
  66     def add_features(self):
  67         if self._config['FEATURES'].getboolean('ephemeris'):
  68             config_file = eval(self._config['FEATURE_CONFIG']['ephemeris'])
  69             with open(config_file) as f:
  70                 self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
  71                 self._config_text += f.read() + '\n\n'
  72
  73             ephemerides = Ephemeris(config_file=config_file, start=self._start, end=self._end)
  74
  75             # ephemerides.start = self._start
  76             # ephemerides.end = self._end
  77
  78             dated_features = ephemerides.dated_features
  79             for date in dated_features:
  80                 self._X.setdefault(date, {}).update(dated_features[date])
  81
  82         if self._config['FEATURES'].getboolean('holidays'):
  83             config_file = eval(self._config['FEATURE_CONFIG']['holidays'])
  84             with open(config_file) as f:
  85                 self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
  86                 self._config_text += f.read() + '\n\n'
  87
  88             holidays = Holidays(config_file=config_file)
  89
  90             holidays.start = self._start
  91             holidays.end = self._end
  92
  93             dated_features = holidays.dated_features
  94             for date in dated_features:
  95                 self._X.setdefault(date, {}).update(dated_features[date])
  96
  97         if self._config['FEATURES'].getboolean('meteofrance'):
  98             config_file = eval(self._config['FEATURE_CONFIG']['meteofrance'])
  99             with open(config_file) as f:
 100                 self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
 101                 self._config_text += f.read() + '\n\n'
 102
 103             meteofeature = MeteoFrance(config_file=config_file)
 104
 105             meteofeature.start = self._start
 106             meteofeature.end = self._end
 107
 108             meteofeature.update()
 109             dated_features = meteofeature.dated_features
 110             for date in dated_features:
 111                 self._X.setdefault(date, {}).update(dated_features[date])
 112
 113         if self._config['FEATURES'].getboolean('ramadan'):
 114             config_file = eval(self._config['FEATURE_CONFIG']['ramadan'])
 115             with open(config_file) as f:
 116                 self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
 117                 self._config_text += f.read() + '\n\n'
 118
 119             ramadan = Ramadan(config_file=config_file)
 120
 121             ramadan.start = self._start
 122             ramadan.end = self._end
 123
 124             dated_features = ramadan.dated_features
 125             for date in dated_features:
 126                 self._X.setdefault(date, {}).update(dated_features[date])
 127
 128     def add_target(self):
 129         config_file = eval(self._config['TARGET']['config'])
 130         cumulative = self._config['TARGET'].getboolean('cumulative')
 131         with open(config_file) as f:
 132             self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
 133             self._config_text += f.read() + '\n\n'
 134
 135         self._target = Target(config_file=config_file,
 136                               start=self._start, end=self._end,
 137                               timestep=self._timestep, cumulative=cumulative)
 138
 139     def add_preprocessing(self):
 140         self._preproc = Preprocessing(config_file=self._config,
 141                                       start=self._start, end=self._end,
 142                                       timestep=self._timestep,
 143                                       dict_features=self.X,
 144                                       dict_target=self.y)
 145
 146     def learn(self):
 147         config_file = eval(self._config['LEARNER']['config'])
 148         with open(config_file) as f:
 149             self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
 150             self._config_text += f.read() + '\n\n'
 151
 152         history = eval(self._config['HISTORY_KNOWLEDGE']['nb_lines'])
 153         self._learner = Learning(config_file=config_file, file_name=self._file_name,
 154                                  X=self._preproc.dataframe, y=list(self.y.values())[history:],
 155                                  horizon=self._config['TARGET'].getint('horizon'))
 156
 157     @property
 158     def X(self):
 159         return self._X
 160
 161     @X.setter
 162     def X(self, x):
 163         self._X = x
 164
 165     @property
 166     def y(self):
 167         return self._target.y
 168
 169     @y.setter
 170     def end(self, y):
 171         self._target.y = y