From 71c4ef80653c05789878229f07ca266a5459cfcf Mon Sep 17 00:00:00 2001 From: fordprefect Date: Thu, 21 Jan 2021 18:42:48 +0100 Subject: [PATCH] various new features: new vaccination overview on country specific side, slightly changed text on index head, moved data correction to parser, made population an int, added the used vaccines to the vaccination table, and some bugfixing here and there --- all_countries.py | 38 ++++++++++++++------ coronavis.py | 56 +++++++++++++++++++++++++++-- country_details.py | 90 ++++++++++++++++++++++++++++++++++++++++++---- index.html.head | 4 +-- 4 files changed, 166 insertions(+), 22 deletions(-) diff --git a/all_countries.py b/all_countries.py index 59aadfa..9c551fc 100644 --- a/all_countries.py +++ b/all_countries.py @@ -23,7 +23,7 @@ corr = {"Chile": 10000, "Spain": 30000, } -def plot(data, countries, pop, **kwargs): +def plot(data, countries, pop, metadata={}, **kwargs): figsize = (10,5) vaccs = [] @@ -33,7 +33,7 @@ def plot(data, countries, pop, **kwargs): if loc == "International": continue name = basename+loc - time, new_cases, new_deaths, total_cases, total_deaths, total_vaccinations, stringency_index = data[loc]['time'], data[loc]['new_cases'], data[loc]['new_deaths'], data[loc]['total_cases'], data[loc]['total_deaths'], data[loc]['total_vaccinations'], data[loc]['stringency_index'] + time, new_cases, new_deaths, total_cases, total_deaths, total_vaccinations, stringency_index, new_vaccinations = data[loc]['time'], data[loc]['new_cases'], data[loc]['new_deaths'], data[loc]['total_cases'], data[loc]['total_deaths'], data[loc]['total_vaccinations'], data[loc]['stringency_index'], data[loc]['new_vaccinations'] fig, ax1 = pp.subplots(num=name, figsize=figsize) @@ -49,16 +49,25 @@ def plot(data, countries, pop, **kwargs): ax1.plot(time[3:-3], np.convolve(new_cases, np.ones((7,))/7, mode="valid"), label="new cases 7day mean", color="orange", linestyle="-", linewidth=2) # plot vaccinations - if not np.isnan(total_vaccinations).all() : - print(f"{loc} has vaccines, adding to plot") - - # fix data: not all countries report daily - for n in range(1, len(total_vaccinations)): - if np.isnan(total_vaccinations[n]) and not np.isnan(total_vaccinations[n-1]): - total_vaccinations[n] = total_vaccinations[n-1] + if not np.isnan(total_vaccinations).all(): + # notify of new vaccine programs + if np.isnan(total_vaccinations[-2]): + print(f"{loc} starts vaccinating, adding to plot") ax2.plot(time, np.array(total_vaccinations), label=f"Total vaccinations", marker="", linestyle="-.", color="crimson") + if False: + # plot detailed vaccination plot for all_countries.py + rfig, rax = pp.subplots(1,1,num=f"{loc}_vacc") + rax2 = rax.twinx() + rax.plot(time, new_vaccinations, linestyle="--", color="green", label="new vac") + rax2.plot(time, total_vaccinations, linestyle="-.", color="red", label="total vac") + rax.set_ylabel("new vaccinatios") + rax2.set_ylabel("total vaccinations") + rax.legend(frameon=False, loc=2) + + rfig.savefig("img/"+f"{loc}".replace(" ", "_").replace("'", "").replace("/", "") + "/vaccs.png") + # fix lower bound of plot for ax in (ax1, ax2): axis = ax.axis() @@ -154,10 +163,17 @@ def plot(data, countries, pop, **kwargs): with open("index.html.head", "r") as g: f.write(g.read()) # table header - f.write("\n") + f.write("
LandImpfungenImpfrate
\n") # data for loc, tvac, rvac in vaccs: - f.write(f"\n".replace(".", ",")) + line = f"".replace(".", ",") + if "vaccines" in metadata[loc]: + line += f"\n" + else: + line += f"\n" + f.write(line) # table footer f.write("
LandImpfungenImpfrateImpfstoffe
{loc}" + f"{tvac:,d}".replace(",",".") + f"{rvac:3.3f}%
{loc}" + \ + f"{tvac:,d}".replace(",",".") + \ + f"{rvac:3.3f}%{metadata[loc]['vaccines']}
-
\n") # site footer diff --git a/coronavis.py b/coronavis.py index 815e105..0dd5a77 100644 --- a/coronavis.py +++ b/coronavis.py @@ -94,6 +94,7 @@ def get_data(): metadata = {} with open(datafile, "r") as f: reader = csv.reader(f) + for row in reader: if len(row) == 6: date,location,new_cases,new_deaths,total_cases,total_deaths = row @@ -105,6 +106,8 @@ def get_data(): iso_code,continent,location,date,total_cases,new_cases,new_cases_smoothed,total_deaths,new_deaths,new_deaths_smoothed,total_cases_per_million,new_cases_per_million,new_cases_smoothed_per_million,total_deaths_per_million,new_deaths_per_million,new_deaths_smoothed_per_million,reproduction_rate,icu_patients,icu_patients_per_million,hosp_patients,hosp_patients_per_million,weekly_icu_admissions,weekly_icu_admissions_per_million,weekly_hosp_admissions,weekly_hosp_admissions_per_million,new_tests,total_tests,total_tests_per_thousand,new_tests_per_thousand,new_tests_smoothed,new_tests_smoothed_per_thousand,positive_rate,tests_per_case,tests_units,total_vaccinations,total_vaccinations_per_hundred,stringency_index,population,population_density,median_age,aged_65_older,aged_70_older,gdp_per_capita,extreme_poverty,cardiovasc_death_rate,diabetes_prevalence,female_smokers,male_smokers,handwashing_facilities,hospital_beds_per_thousand,life_expectancy,human_development_index = row elif len(row) == 54: iso_code, continent, location, date, total_cases, new_cases, new_cases_smoothed, total_deaths, new_deaths, new_deaths_smoothed, total_cases_per_million, new_cases_per_million, new_cases_smoothed_per_million, total_deaths_per_million, new_deaths_per_million, new_deaths_smoothed_per_million, reproduction_rate, icu_patients, icu_patients_per_million, hosp_patients, hosp_patients_per_million, weekly_icu_admissions, weekly_icu_admissions_per_million, weekly_hosp_admissions, weekly_hosp_admissions_per_million, new_tests, total_tests, total_tests_per_thousand, new_tests_per_thousand, new_tests_smoothed, new_tests_smoothed_per_thousand, positive_rate, tests_per_case, tests_units, total_vaccinations, new_vaccinations, total_vaccinations_per_hundred, new_vaccinations_per_million, stringency_index, population, population_density, median_age, aged_65_older, aged_70_older, gdp_per_capita, extreme_poverty, cardiovasc_death_rate, diabetes_prevalence, female_smokers, male_smokers, handwashing_facilities, hospital_beds_per_thousand, life_expectancy, human_development_index = row + elif len(row) == 55: + iso_code, continent, location, date, total_cases, new_cases, new_cases_smoothed, total_deaths, new_deaths, new_deaths_smoothed, total_cases_per_million, new_cases_per_million, new_cases_smoothed_per_million, total_deaths_per_million, new_deaths_per_million, new_deaths_smoothed_per_million, reproduction_rate, icu_patients, icu_patients_per_million, hosp_patients, hosp_patients_per_million, weekly_icu_admissions, weekly_icu_admissions_per_million, weekly_hosp_admissions, weekly_hosp_admissions_per_million, new_tests, total_tests, total_tests_per_thousand, new_tests_per_thousand, new_tests_smoothed, new_tests_smoothed_per_thousand, positive_rate, tests_per_case, tests_units, total_vaccinations, new_vaccinations, new_vaccinations_smoothed, total_vaccinations_per_hundred, new_vaccinations_smoothed_per_million, stringency_index, population, population_density, median_age, aged_65_older, aged_70_older, gdp_per_capita, extreme_poverty, cardiovasc_death_rate, diabetes_prevalence, female_smokers, male_smokers, handwashing_facilities, hospital_beds_per_thousand, life_expectancy, human_development_index = row else: print(f"WARNING! Table format changed, length now {len(row)}, new header:\n{row})") exit(1) @@ -130,6 +133,7 @@ def get_data(): total_tests = tofloat(total_tests) positive_rate = tofloat(positive_rate) tests_per_case = tofloat(tests_per_case) + new_vaccinations = tofloat(new_vaccinations) tests_units = tests_units if location not in data: @@ -141,7 +145,8 @@ def get_data(): new_cases, new_deaths, total_cases, total_deaths, total_vaccinations, stringency_index, reproduction_rate, icu_patients, hosp_patients, weekly_icu_admissions, weekly_hosp_admissions, new_tests, - total_tests, positive_rate, tests_per_case, tests_units,] + total_tests, positive_rate, tests_per_case, tests_units, + new_vaccinations,] ) @@ -155,6 +160,21 @@ def get_data(): else: if metadata[location][field] != row[n]: print(f"{location}: {field} seems not to be a constant ({metadata[location][field]} vs {row[n]})") + + ### End of csv reading loop + +# get data about vaccines + vaccinesurl = "https://github.com/owid/covid-19-data/raw/master/public/data/vaccinations/locations.csv" + vacraw = requests.get(vaccinesurl).content.decode("UTF8").split('\n')[1:-1] + vacreader = csv.reader(vacraw) + vaccines_country_dict = {} + + for row in vacreader: + land = row[0] + vaccines = row[2] + vaccines_country_dict[land] = vaccines + del(vaccinesurl, vacraw, vacreader) + # reorganize data data2 = {} for loc in data: @@ -170,8 +190,9 @@ def get_data(): positive_rate = [] tests_per_case = [] tests_units = [] + new_vaccinations = [] for entry in data[loc]: - t_, new_cases_, new_deaths_, total_cases_, total_deaths_, total_vaccinations_, stringency_index_, reproduction_rate_, icu_patients_, hosp_patients_, weekly_icu_admissions_, weekly_hosp_admissions_, new_tests_, total_tests_, positive_rate_, tests_per_case_, tests_units_ = entry + t_, new_cases_, new_deaths_, total_cases_, total_deaths_, total_vaccinations_, stringency_index_, reproduction_rate_, icu_patients_, hosp_patients_, weekly_icu_admissions_, weekly_hosp_admissions_, new_tests_, total_tests_, positive_rate_, tests_per_case_, tests_units_, new_vaccinations_ = entry time.append(t_) new_cases.append(toint(new_cases_)) @@ -189,7 +210,21 @@ def get_data(): total_tests.append(toint(total_tests_)) positive_rate.append(positive_rate_) tests_per_case.append(tests_per_case_) + new_vaccinations.append(toint(new_vaccinations_)) tests_units.append(tests_units_) + + ### data tweaking and fixing goes here + + # fix vaccination data: not all countries report daily vaccinations + for n in range(1, len(total_vaccinations)): + if np.isnan(total_vaccinations[n]) and not np.isnan(total_vaccinations[n-1]): + total_vaccinations[n] = total_vaccinations[n-1] + + + ### + + + # collecting data data2[loc] = {'time': time, 'new_cases': new_cases, 'new_deaths': new_deaths, @@ -207,11 +242,28 @@ def get_data(): 'positive_rate': positive_rate, 'tests_per_case': tests_per_case, 'tests_units': tests_units, + 'new_vaccinations': new_vaccinations, } + # add vaccine info to metadata + if loc in vaccines_country_dict: + metadata[loc]['vaccines'] = vaccines_country_dict[loc] + # cast population to int + if loc != "International": + try: metadata[loc]['population'] = int(float(metadata[loc]['population'])) + except: metadata[loc][loc]['population'] = np.nan + return data2, metadata data, metadata = get_data() +## dump data instead of plotting +if False: + print("dumping data, no plotting") + import pickle + with open("data.dump", "wb") as f: + pickle.dump([data, metadata], f) + exit() + for plot in plots: i = importlib.import_module(plot) i.plot(data, countries, pop, metadata=metadata) diff --git a/country_details.py b/country_details.py index 1ca3310..6475d69 100644 --- a/country_details.py +++ b/country_details.py @@ -9,6 +9,9 @@ import pickle import logging logging.getLogger().setLevel(logging.CRITICAL) import os +import datetime +import locale +locale.setlocale(locale.LC_ALL, "de_DE.utf8") def plot(data, countries, pop, metadata, **kwargs): figsize = (10,5) @@ -24,7 +27,31 @@ def plot(data, countries, pop, metadata, **kwargs): if not os.path.isdir(path): os.mkdir(path) - if not os.path.isfile(path+"/index.html") or True: # TODO enable html file generation + # is this country vaccinating? + is_vaccinating = True if data[loc]['total_vaccinations'][-1] > 0 else False + if is_vaccinating: + first_vac_report = np.argwhere(np.isnan(data[loc]['total_vaccinations']))[-1][0] + 1 + vac_text = f"Impfstart{data[loc]['time'][first_vac_report]}" + try: + vac_rate = data[loc]['total_vaccinations'][-1]/metadata[loc]['population']*100 + vac_text+= f"Impfrate{vac_rate:1.3f} %" + except: + pass + try: + immune_rate = (data[loc]['total_vaccinations'][-1] + data[loc]['total_cases'][-1])/metadata[loc]['population']*100 + vac_text += f"Immunrate{immune_rate:1.3f} %" + except: + pass + try: + vaccines = metadata[loc]['vaccines'] if 'vaccines' in metadata[loc] else "unknown" + vac_text += f"Impfstoffe{vaccines}" + except: + pass + total_cases = data[loc]['total_cases'][-1] + total_deaths = data[loc]['total_deaths'][-1] + today = datetime.datetime.now().strftime("%d.%m.%Y") + + if True: #not os.path.isfile(path+"/index.html") or False: # TODO enable html file generation with open(path+"/index.html", "w") as f: f.write(f""" @@ -42,6 +69,7 @@ def plot(data, countries, pop, metadata, **kwargs): Zurück

Landesspezifische Kennzahlen

 +(Stand: {today}) @@ -61,12 +89,18 @@ def plot(data, countries, pop, metadata, **kwargs): + + +{vac_text if is_vaccinating else ''}
ISO Code{metadata[loc]["iso_code"]}
Continent{metadata[loc]["continent"]}
hospital beds per thousand inhabitants{metadata[loc]["hospital_beds_per_thousand"]}
life expectancy{metadata[loc]["life_expectancy"]}
human development index{metadata[loc]["human_development_index"]}
Absolute bestätigte Infektionsfälle{total_cases}
Absolute bestätigte Todesfälle{total_deaths}
Übersicht
Krankenhaussituation
-
Testsituation
+
Testsituation +
+Testing dataset: Hasell, J., Mathieu, E., Beltekian, D. et al. A cross-country database of COVID-19 testing. Sci Data 7, 345 (2020). DOI:10.1038/s41597-020-00688-8
+
Impfsituation


Zurück
@@ -81,6 +115,7 @@ Ein Infoservice von dukun.de; Anregungen gern dukun.de; Anregungen gern sehr unterschiedlich sein!

-Die Daten stammen von hier und werden dort aus den WHO- und ECDC-Reports generiert. -Von den extrem reichhaltigen Daten dort verarbeite ich nur die Zahl der Neufälle. +Die Daten stammen von hier und werden dort aus verschiedensten Quellen aggregiert. +

Aktuelle Daten aus Deutschland mit vielen Hintergründen finden sich im Lagebericht des RKI.