- def __insert_features(self):
- logger.info('Inserting MeteoFrance list of features from meteo_features.csv')
- csv_file = Path.cwd() / 'config' / 'features' / 'meteofrance' / 'meteofrance_features.csv'
- with PostgreSQLDBConnection.Instance() as db:
- with open(csv_file, "r") as f:
- reader = DictReader(f, delimiter=',')
- next(reader)
- for row in reader:
- request = f"""INSERT INTO "METEO_FEATURE" ("METFT_NAME", "PARAM_ID_PARAMETER")
- VALUES ('{row['METFT_NAME']}', {row['PARAM_ID_PARAMETER']});"""
- db.cursor.execute(request)
-
-
-
- def __collect_historical_data(self):
- '''
- We collect all csv files from January 1996 until the month
- before now. The argument in the url to download are of the
- form 201001 for January 2010. We start by computing all these
- patterns, in historical list.
- '''
- # List of year-months to consider
- historical = []
- date_end = datetime.now()
- for year in range(1996, date_end.year+1):
- for month in range(1,13):
- date = datetime(year, month, 1)
- if date <= date_end:
- historical.append(date.strftime("%Y%m"))
-
- # We download all csv files from meteofrance that are not in
- # the data repository
- meteo_data = self.__data_directory / 'historical'
- p = Path(meteo_data)
- p.mkdir(exist_ok=True, parents=True)
- for date in historical:
- if not isfile(meteo_data / ('synop.'+date+'.csv')):
- link = 'https://donneespubliques.meteofrance.fr/donnees_libres/Txt/Synop/Archive/synop.'
- link += date + '.csv.gz'
- download_path = meteo_data / basename(link)
- urlretrieve(link, download_path)
- with gzip.open(download_path, 'rb') as f:
- csv_file = meteo_data / basename(link[:-3])
- with open(csv_file, 'w') as g:
- g.write(f.read().decode())
- remove(meteo_data / basename(link))
-
-
- def __from_date_to_datetz(self, date, a, b):
- if not hasattr(self, '__meteo_station_tz'):
- self.__meteo_station_tz = {}
- tf = TimezoneFinder(in_memory=True)
- with PostgreSQLDBConnection.Instance() as db:
- db.cursor.execute('select "METST_IDNAME", "METST_LOCATION" from "METEO_STATION";')
- list_of_rows = db.cursor.fetchall()
- for k in list_of_rows:
- print('\n',k)
- longitude, latitude = eval(k[1])
- print(longitude, latitude)
- print(type(longitude))
- timezone_name = tf.timezone_at(lng = longitude, lat = latitude)
- if timezone_name is None:
- timezone_name = tf.closest_timezone_at(lng = longitude,
- lat = latitude,
- delta_degree = 13,
- exact_computation=True,
- #return_distances=True,
- force_evaluation=True)
- cet = pytz.timezone(timezone_name)
- dt = datetime.now()
- offset = cet.utcoffset(dt, is_dst = True)
- shift = int(offset / timedelta(hours=1))
- self.__meteo_station_tz[k[0]] = shift
