predictops/engine.py

   1 from configparser import ConfigParser
   2 from datetime import datetime, timedelta
   3 from logging import getLogger
   4 from logging.config import fileConfig
   5 from pathlib import Path
   6 from shutil import rmtree
   7
   8 import os
   9
  10 from .learn.learning import Learning
  11 from .learn.preprocessing import Preprocessing
  12 from .source.ephemeris import Ephemeris
  13 from .source.holidays import Holidays
  14 from .source.ramadan import Ramadan
  15 from .source.meteofrance import MeteoFrance
  16 from .target.target import Target
  17
  18 fileConfig((Path.cwd() / 'config') / 'logging.cfg')
  19 logger = getLogger()
  20
  21
  22 class Engine(object):
  23
  24     def __enter__(self):
  25         return self
  26
  27     def __exit__(self, type, value, traceback):
  28         with open(str(self._file_name / os.path.basename(self._file_name)) + '.cfg', 'w') as f:
  29             f.write(self._config_text)
  30
  31     def __init__(self, config_file=(Path.cwd() / 'config') / 'learn.cfg'):
  32         self._config = ConfigParser()
  33         self._config.read(config_file)
  34         launching_time = datetime.strftime(datetime.now(), '%Y_%m_%d_%H_%M')
  35         self._name = os.path.splitext(os.path.basename(eval(self._config['TARGET']['config'])))[0]
  36         self._file_name = f"{self._name}-{launching_time}"
  37         p = Path.cwd() / 'results' / self._name
  38         p.mkdir(exist_ok=True, parents=True)
  39         self._file_name = p / self._file_name
  40
  41         self._config_text = ''
  42         with open(config_file) as f:
  43             self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
  44             self._config_text += f.read() + '\n\n'
  45
  46         self._start = datetime.strptime(self._config['DATETIME']['start'],
  47                                         '%m/%d/%Y %H:%M:%S')
  48         self._end = datetime.strptime(self._config['DATETIME']['end'],
  49                                       '%m/%d/%Y %H:%M:%S')
  50
  51         self._timestep = timedelta(hours=self._config['DATETIME'].getfloat('hourStep'))
  52
  53         self._X = {}
  54
  55     def clean(self):
  56         # Cleaning the data directory
  57         logger.info("Cleaning and restoring data directory")
  58         directory = Path.cwd() / 'data'
  59         if directory.is_dir():
  60             rmtree(directory)
  61         p = Path(Path.cwd() / 'data')
  62         p.mkdir()
  63
  64     def add_features(self):
  65         if self._config['FEATURES'].getboolean('ephemeris'):
  66             config_file = eval(self._config['FEATURE_CONFIG']['ephemeris'])
  67             with open(config_file) as f:
  68                 self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
  69                 self._config_text += f.read() + '\n\n'
  70
  71             ephemerides = Ephemeris(config_file=config_file)
  72
  73             ephemerides.start = self._start
  74             ephemerides.end = self._end
  75
  76             dated_features = ephemerides.dated_features
  77             for date in dated_features:
  78                 self._X.setdefault(date, {}).update(dated_features[date])
  79
  80         if self._config['FEATURES'].getboolean('holidays'):
  81             config_file = eval(self._config['FEATURE_CONFIG']['holidays'])
  82             with open(config_file) as f:
  83                 self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
  84                 self._config_text += f.read() + '\n\n'
  85
  86             holidays = Holidays(config_file=config_file)
  87
  88             holidays.start = self._start
  89             holidays.end = self._end
  90
  91             dated_features = holidays.dated_features
  92             for date in dated_features:
  93                 self._X.setdefault(date, {}).update(dated_features[date])
  94
  95         if self._config['FEATURES'].getboolean('meteofrance'):
  96             config_file = eval(self._config['FEATURE_CONFIG']['meteofrance'])
  97             with open(config_file) as f:
  98                 self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
  99                 self._config_text += f.read() + '\n\n'
 100
 101             meteofeature = MeteoFrance(config_file=config_file)
 102
 103             meteofeature.start = self._start
 104             meteofeature.end = self._end
 105
 106             meteofeature.update()
 107             dated_features = meteofeature.dated_features
 108             for date in dated_features:
 109                 self._X.setdefault(date, {}).update(dated_features[date])
 110
 111         if self._config['FEATURES'].getboolean('ramadan'):
 112             config_file = eval(self._config['FEATURE_CONFIG']['ramadan'])
 113             with open(config_file) as f:
 114                 self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
 115                 self._config_text += f.read() + '\n\n'
 116
 117             ramadan = Ramadan(config_file=config_file)
 118
 119             ramadan.start = self._start
 120             ramadan.end = self._end
 121
 122             dated_features = ramadan.dated_features
 123             for date in dated_features:
 124                 self._X.setdefault(date, {}).update(dated_features[date])
 125
 126     def add_target(self):
 127         config_file = eval(self._config['TARGET']['config'])
 128         cumulative = self._config['TARGET'].getboolean('cumulative')
 129         with open(config_file) as f:
 130             self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
 131             self._config_text += f.read() + '\n\n'
 132
 133         self._target = Target(config_file=config_file,
 134                               start=self._start, end=self._end,
 135                               timestep=self._timestep, cumulative=cumulative)
 136
 137     def add_preprocessing(self):
 138         self._preproc = Preprocessing(config_file=self._config,
 139                                       dict_features=self.X,
 140                                       dict_target=self.y)
 141
 142     def learn(self):
 143         config_file = eval(self._config['LEARNER']['config'])
 144         with open(config_file) as f:
 145             self._config_text += f"{'='*10} {os.path.basename(config_file)} {'='*10}\n\n"
 146             self._config_text += f.read() + '\n\n'
 147
 148         history = eval(self._config['HISTORY_KNOWLEDGE']['nb_lines'])
 149         self._learner = Learning(config_file=config_file, file_name=self._file_name,
 150                                  X=self._preproc.dataframe, y=list(self.y.values())[history:],
 151                                  horizon=self._config['TARGET'].getint('horizon'))
 152
 153     @property
 154     def X(self):
 155         return self._X
 156
 157     @X.setter
 158     def X(self, x):
 159         self._X = x
 160
 161     @property
 162     def y(self):
 163         return self._target.y
 164
 165     @y.setter
 166     def end(self, y):
 167         self._target.y = y