X-Git-Url: https://bilbo.iut-bm.univ-fcomte.fr/and/gitweb/predictops.git/blobdiff_plain/90e69cb2125d4bae76a27b9c38defb4f70bf2ca6..refs/heads/master:/predictops/source/meteofrance.py?ds=inline diff --git a/predictops/source/meteofrance.py b/predictops/source/meteofrance.py index 0edd49f..05eb710 100644 --- a/predictops/source/meteofrance.py +++ b/predictops/source/meteofrance.py @@ -19,12 +19,12 @@ logger = getLogger() class MeteoFrance: - _latitude = None - _longitude = None + _latitude = None + _longitude = None _nb_stations = None - _start = None - _end = None - _features = None + _start = None + _end = None + _features = None def __init__(self, config_file): ''' @@ -51,10 +51,9 @@ class MeteoFrance: # Collecting meteofrance features self._features = [section for section in self._config if self._config.has_option(section, 'numerical') - and (self._config[section]['numerical'] or - self._config[section]['categorical'])] - - + and (self._config[section]['binary'] or + self._config[section]['categorical'] or + self._config[section]['numerical'])] @property def start(self): @@ -64,7 +63,6 @@ class MeteoFrance: def start(self, x): self._start = x - @property def end(self): return self._end @@ -73,7 +71,6 @@ class MeteoFrance: def end(self, x): self._end = x - @property def latitude(self): return self._latitude @@ -82,7 +79,6 @@ class MeteoFrance: def latitude(self, x): self._latitude = x - @property def longitude(self): return self._longitude @@ -91,7 +87,6 @@ class MeteoFrance: def longitude(self, x): self._longitude = x - @property def nb_stations(self): return self._nb_stations @@ -100,7 +95,6 @@ class MeteoFrance: def nb_stations(self, x): self._nb_stations = x - def _regenerate_directory(self): ''' Re-creating data directory architecture for MeteoFrance @@ -115,8 +109,6 @@ class MeteoFrance: p = Path(self._data_directory / 'config') p.mkdir(exist_ok=True, parents=True) - - def _get_stations(self): ''' Collect (after downloading them, if needed) the stations and their @@ -129,7 +121,7 @@ class MeteoFrance: # The csv file of meteo stations (names, ids and locations) if downloaded, # if not available in the config directory within data / meteo_france link = 'https://donneespubliques.meteofrance.fr/donnees_libres/Txt/Synop/postesSynop.csv' - p = Path(self._data_directory / 'config' ) + p = Path(self._data_directory / 'config') csv_file = p / basename(link) if not isfile(csv_file): logger.info('Downloading location stations from MeteoFrance') @@ -142,11 +134,11 @@ class MeteoFrance: reader = DictReader(f, delimiter=';') for row in reader: latitude, longitude = eval(row['Latitude']), eval(row['Longitude']) - self._dict_stations[row['Nom'].replace("'",'’')] = { - 'id' : row['ID'], - 'longitude' : longitude, - 'latitude' : latitude, - 'distance' : vincenty( + self._dict_stations[row['Nom'].replace("'", '’')] = { + 'id': row['ID'], + 'longitude': longitude, + 'latitude': latitude, + 'distance': vincenty( (self._latitude, self._longitude), (latitude, longitude)).km } @@ -154,13 +146,11 @@ class MeteoFrance: # Find the closest stations logger.info('Finding the closest stations') stations_by_distance = sorted(self._dict_stations.keys(), - key = lambda x: self._dict_stations[x]['distance']) + key=lambda x: self._dict_stations[x]['distance']) logger.info(f'The {self._nb_stations} closest stations are: ' f'{", ".join(stations_by_distance[:self._nb_stations])}.') return [self._dict_stations[sta]['id'] for sta in stations_by_distance][:self._nb_stations] - - def _collect_historical_data(self): ''' We collect all csv files from January 1996 until the month @@ -171,8 +161,8 @@ class MeteoFrance: # List of year-months to consider historical = [] date_end = self._end - for year in range(self._start.year, date_end.year+1): - for month in range(1,13): + for year in range(self._start.year, date_end.year + 1): + for month in range(1, 13): date = datetime(year, month, 1) if date >= self._start and date <= date_end: historical.append(date.strftime("%Y%m")) @@ -183,7 +173,7 @@ class MeteoFrance: p = Path(meteo_data) p.mkdir(exist_ok=True, parents=True) for date in historical: - if not isfile(meteo_data / ('synop.'+date+'.csv')): + if not isfile(meteo_data / ('synop.' + date + '.csv')): link = 'https://donneespubliques.meteofrance.fr/donnees_libres/Txt/Synop/Archive/synop.' link += date + '.csv.gz' download_path = meteo_data / basename(link) @@ -194,8 +184,6 @@ class MeteoFrance: g.write(f.read().decode()) remove(meteo_data / basename(link)) - - def update(self): ''' Update the MeteoFrance features with the last available data @@ -209,15 +197,13 @@ class MeteoFrance: logger.info('Update historical csv files from MeteoFrance, if needed') today = datetime.now() - todel = 'synop.'+today.strftime("%Y%m")+".csv" + todel = 'synop.' + today.strftime("%Y%m") + ".csv" try: remove(self._data_directory / 'historical' / todel) except: logger.warning(f"{self._data_directory / 'historical' / todel} not found") self._collect_historical_data() - - @property def dated_features(self): ''' @@ -234,26 +220,27 @@ class MeteoFrance: logger.info('Collecting meteofrance feature information') # A dictionary for the features dico_features = {self._config[section]["abbreviation"]: - { - 'name': section, # feature name - 'numerical': self._config[section]['numerical'], - 'categorical': self._config[section]['categorical'] - } - for section in self._features} + { + 'name': section, # feature name + 'numerical': self._config[section]['numerical'], + 'categorical': self._config[section]['categorical'] + } + for section in self._features} dir_data = Path.cwd() / 'data' / 'features' / 'meteo_france' / 'historical' self._dated_features = {} + for csv_meteo in sorted(listdir(dir_data)): date = datetime.strptime(csv_meteo.split('.')[1], '%Y%m') if (date >= self._start and date <= self._end)\ - or (date.year == self._start.year and date.month == self._start.month)\ - or (date.year == self._end.year and date.month == self._end.month): + or (date.year == self._start.year and date.month == self._start.month)\ + or (date.year == self._end.year and date.month == self._end.month): logger.info(f'Adding meteofrance features from {csv_meteo}') with open(dir_data / csv_meteo, "r") as f: reader = DictReader(f, delimiter=';') for row in reader: if row['numer_sta'] in self._stations: date = datetime.strptime(row['date'], '%Y%m%d%H%M%S') - if date >= self._start and date <= self._end: - self._dated_features.setdefault(date,{}).update({dico_features[feat]['name']+'_'+str(self._stations.index(row['numer_sta'])): eval(row[feat].replace('mq','None')) for feat in dico_features}) + if date >= self._start and date <= self._end: + self._dated_features.setdefault(date, {}).update({dico_features[feat]['name'] + '_' + str(self._stations.index(row['numer_sta'])): eval(row[feat].replace('mq', 'None')) for feat in dico_features}) return self._dated_features