Standardization and one hot encoding

diff --git a/main.py b/main.py
index 969054c7ce186040b11a918c079a32b3a240cf94..d1d7f9c24721819d8e9d0a39eb50a1eb52546e47 100644 (file)
--- a/main.py
+++ b/main.py
@@ -5,7 +5,6 @@ from logging import getLogger
 from logging.config import fileConfig
 from pathlib import Path
 
 from logging.config import fileConfig
 from pathlib import Path
 

-import geopandas as gpd

 
 fileConfig((Path.cwd() / 'config') / 'logging.cfg')
 logger = getLogger()
 
 fileConfig((Path.cwd() / 'config') / 'logging.cfg')
 logger = getLogger()


@@ -23,14 +22,11 @@ if __name__ == '__main__':
     print(process.dataframe.head(n=20))
     print(process.dataframe.tail(n=20))
 
     print(process.dataframe.head(n=20))
     print(process.dataframe.tail(n=20))
 

-    #target = toarea(stream_file = Path.cwd() / 'data' / 'targets' / 'sdis25' / 'interventions.csv')
-
-
-    exit()
+    '''target = toarea(stream_file = Path.cwd() / 'data' / 'targets' / 'sdis25' / 'interventions.csv')

 
     depts = gpd.read_file( Path.cwd() / 'data' / 'targets' / 'departments' / "departements-20180101.shp")
     Doubs = depts.loc[depts['nom'] == 'Doubs'].iloc[0]
 
     ToArea(area=Doubs.geometry,
            csv_file = Path.cwd() / 'data' / 'targets' / 'sdis25' / 'interventions.csv')
 
     depts = gpd.read_file( Path.cwd() / 'data' / 'targets' / 'departments' / "departements-20180101.shp")
     Doubs = depts.loc[depts['nom'] == 'Doubs'].iloc[0]
 
     ToArea(area=Doubs.geometry,
            csv_file = Path.cwd() / 'data' / 'targets' / 'sdis25' / 'interventions.csv')

-
+    '''

diff --git a/predictops/learn/preprocessing.py b/predictops/learn/preprocessing.py
index 939a7fa30e79d45314adec6f8d526362137e3e9b..4197b8fed13cb4137e33655753976532e42987a2 100644 (file)
--- a/predictops/learn/preprocessing.py
+++ b/predictops/learn/preprocessing.py
@@ -6,6 +6,7 @@ from logging import getLogger
 from logging.config import fileConfig
 from os import listdir
 from pathlib import Path
 from logging.config import fileConfig
 from os import listdir
 from pathlib import Path

+from sklearn import preprocessing

 
 import numpy as np
 import pandas as pd
 
 import numpy as np
 import pandas as pd


@@ -134,6 +135,75 @@ class Preprocessing:
         return self._full_dict
 
 
         return self._full_dict
 
 

+    def _fill_nan(self):
+        '''
+        Fill NaN values, either by propagation or by interpolation (linear or splines)
+        '''
+        logger.info("Filling NaN numerical values in the feature dataframe")
+        # We interpolate (linearly or with splines) only numerical columns
+        numerical_columns = [k for k in self._features if self._features[k]['type'] == 1
+                   or (self._features[k]['type'] == 3 and self._features[k]['numerical'])]
+        # The interpolation
+        if self._config['PREPROCESSING']['fill_method'] == 'propagate':
+            self._dataframe[numerical_columns] =\
+                self._dataframe[numerical_columns].fillna(method='ffill')
+        elif self._config['PREPROCESSING']['fill_method'] == 'linear':
+            self._dataframe[numerical_columns] =\
+                self._dataframe[numerical_columns].interpolate()
+        elif self._config['PREPROCESSING']['fill_method'] == 'spline':
+            self._dataframe[numerical_columns] =\
+                self._dataframe[numerical_columns].interpolate(method='spline',
+                     order=self._config['PREPROCESSING'].getint('order'))
+
+        # For the categorical columns, NaN values are filled by duplicating
+        # the last known value (forward fill method)
+        logger.info("Filling NaN categorical values in the feature dataframe")
+        categorical_columns = [k for k in self._features if self._features[k]['type'] == 2
+                   or (self._features[k]['type'] == 3 and not self._features[k]['numerical'])]
+        self._dataframe[categorical_columns] =\
+            self._dataframe[categorical_columns].fillna(method='ffill')
+
+        # Uncomment this line to fill NaN values at the beginning of the
+        # dataframe. This may not be a good idea, especially for features
+        # that are available only for recent years, e.g., air quality
+        #self._dataframe = self._dataframe.fillna(method='bfill')
+
+        # Dropping rows that are not related to our datetime window (start/
+        # step / end)
+        self._dataframe = self._dataframe.drop([k.to_pydatetime()
+                                               for k in self._dataframe.T
+                                               if k not in self._datetimes])
+
+
+    def _standardize(self):
+        '''
+        Normalizing numerical features
+        '''
+        logger.info("Standardizing numerical values in the feature dataframe")
+        # We operate only on numerical columns
+        numerical_columns = [k for k in self._features if self._features[k]['type'] == 1
+                   or (self._features[k]['type'] == 3 and self._features[k]['numerical'])]
+        self._dataframe[numerical_columns] = preprocessing.scale(self._dataframe[numerical_columns])
+
+
+    def _one_hot_encoding(self):
+        '''
+        Apply a one hot encoding for category features
+        '''
+        logger.info("One hot encoding for categorical feature")
+        categorical_columns = [k for k in self._features if self._features[k]['type'] == 2
+                   or (self._features[k]['type'] == 3 and not self._features[k]['numerical'])]
+
+        # On fait un codage disjonctif complet des variables qualitatives
+        df_out = pd.DataFrame()
+        for col in categorical_columns:
+            pd1 = pd.get_dummies(self._dataframe[col],prefix=col)
+            for col1 in pd1.columns:
+                df_out[col1] = pd1[col1]
+        self._dataframe = df_out
+        print(self._dataframe.head())
+
+

 
     @property
     def dataframe(self):
 
     @property
     def dataframe(self):


@@ -144,40 +214,12 @@ class Preprocessing:
             logger.info("Creating feature dataframe from feature dictionary")
             self._dataframe = pd.DataFrame.from_dict(self.full_dict,
                                                      orient='index')
             logger.info("Creating feature dataframe from feature dictionary")
             self._dataframe = pd.DataFrame.from_dict(self.full_dict,
                                                      orient='index')

-            logger.info("Filling NaN numerical values in the feature dataframe")
-            # We interpolate (linearly or with splines) only numerical columns
-            numerical_columns = [k for k in self._features if self._features[k]['type'] == 1
-                       or (self._features[k]['type'] == 3 and self._features[k]['numerical'])]
-            # The interpolation
-            if self._config['PREPROCESSING']['fill_method'] == 'propagate':
-                self._dataframe[numerical_columns] =\
-                    self._dataframe[numerical_columns].fillna(method='ffill')
-            elif self._config['PREPROCESSING']['fill_method'] == 'linear':
-                self._dataframe[numerical_columns] =\
-                    self._dataframe[numerical_columns].interpolate()
-            elif self._config['PREPROCESSING']['fill_method'] == 'spline':
-                self._dataframe[numerical_columns] =\
-                    self._dataframe[numerical_columns].interpolate(method='spline',
-                         order=self._config['PREPROCESSING'].getint('order'))
-
-            # For the categorical columns, NaN values are filled by duplicating
-            # the last known value (forward fill method)
-            logger.info("Filling NaN categorical values in the feature dataframe")
-            categorical_columns = [k for k in self._features if self._features[k]['type'] == 2
-                       or (self._features[k]['type'] == 3 and not self._features[k]['numerical'])]
-            self._dataframe[categorical_columns] =\
-                self._dataframe[categorical_columns].fillna(method='ffill')
-
-            # Uncomment this line to fill NaN values at the beginning of the
-            # dataframe. This may not be a good idea, especially for features
-            # that are available only for recent years, e.g., air quality
-            #self._dataframe = self._dataframe.fillna(method='bfill')
-
-            # Dropping rows that are not related to our datetime window (start/
-            # step / end)
-            self._dataframe = self._dataframe.drop([k.to_pydatetime()
-                                                   for k in self._dataframe.T
-                                                   if k not in self._datetimes])
+            # Dealing with NaN values
+            self._fill_nan()
+            # Normalizing numerical values
+            self._standardize()
+            # Dealing with categorical features
+            self._one_hot_encoding()

         return self._dataframe
 
 
         return self._dataframe
 
 

diff --git a/requirements.txt b/requirements.txt
index 20534697e87960fae898e455d3f93405715f955e..6249a161c878fe217583f52d727df621886c4b38 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,12 +6,14 @@ Fiona==1.8.13
 geographiclib==1.50
 geopandas==0.6.3
 geopy==1.21.0
 geographiclib==1.50
 geopandas==0.6.3
 geopy==1.21.0

+joblib==0.14.1

 munch==2.5.0
 numpy==1.18.1
 pandas==1.0.1
 pyproj==2.4.2.post1
 python-dateutil==2.8.1
 pytz==2019.3
 munch==2.5.0
 numpy==1.18.1
 pandas==1.0.1
 pyproj==2.4.2.post1
 python-dateutil==2.8.1
 pytz==2019.3

+scikit-learn==0.22.1

 scipy==1.4.1
 Shapely==1.7.0
 six==1.14.0
 scipy==1.4.1
 Shapely==1.7.0
 six==1.14.0