various new features: new vaccination overview on country specific side, slightly changed text on index head, moved data correction to parser, made population an int, added the used vaccines to the vaccination table, and some bugfixing here and there

all_countries.py

@@ -23,7 +23,7 @@ corr = {"Chile": 10000,
         "Spain": 30000,
 }
 
-def plot(data, countries, pop, **kwargs):
+def plot(data, countries, pop, metadata={}, **kwargs):
     figsize = (10,5)
     
     vaccs = []
@@ -33,7 +33,7 @@ def plot(data, countries, pop, **kwargs):
             if loc == "International":
                 continue
             name = basename+loc
-            time, new_cases, new_deaths, total_cases, total_deaths, total_vaccinations, stringency_index = data[loc]['time'], data[loc]['new_cases'], data[loc]['new_deaths'], data[loc]['total_cases'], data[loc]['total_deaths'], data[loc]['total_vaccinations'], data[loc]['stringency_index']
+            time, new_cases, new_deaths, total_cases, total_deaths, total_vaccinations, stringency_index, new_vaccinations = data[loc]['time'], data[loc]['new_cases'], data[loc]['new_deaths'], data[loc]['total_cases'], data[loc]['total_deaths'], data[loc]['total_vaccinations'], data[loc]['stringency_index'], data[loc]['new_vaccinations']
             
             fig, ax1 = pp.subplots(num=name, figsize=figsize)
 
@@ -49,16 +49,25 @@ def plot(data, countries, pop, **kwargs):
             ax1.plot(time[3:-3], np.convolve(new_cases, np.ones((7,))/7, mode="valid"), label="new cases 7day mean", color="orange", linestyle="-", linewidth=2)
 
             # plot vaccinations
-            if not np.isnan(total_vaccinations).all() :
+            if not np.isnan(total_vaccinations).all():
-                print(f"{loc} has vaccines, adding to plot")
+                # notify of new vaccine programs
-                
+                if np.isnan(total_vaccinations[-2]):
-                # fix data: not all countries report daily
+                    print(f"{loc} starts vaccinating, adding to plot")
-                for n in range(1, len(total_vaccinations)):
-                    if np.isnan(total_vaccinations[n]) and not np.isnan(total_vaccinations[n-1]):
-                        total_vaccinations[n] = total_vaccinations[n-1]
 
                 ax2.plot(time, np.array(total_vaccinations), label=f"Total vaccinations", marker="", linestyle="-.", color="crimson")
 
+                if False:
+                    # plot detailed vaccination plot for all_countries.py
+                    rfig, rax = pp.subplots(1,1,num=f"{loc}_vacc")
+                    rax2 = rax.twinx()
+                    rax.plot(time, new_vaccinations, linestyle="--", color="green", label="new vac")
+                    rax2.plot(time, total_vaccinations, linestyle="-.", color="red", label="total vac")
+                    rax.set_ylabel("new vaccinatios")
+                    rax2.set_ylabel("total vaccinations")
+                    rax.legend(frameon=False, loc=2)
+
+                    rfig.savefig("img/"+f"{loc}".replace(" ", "_").replace("'", "").replace("/", "") + "/vaccs.png")
+
             # fix lower bound of plot
             for ax in (ax1, ax2):
                 axis = ax.axis()
@@ -154,10 +163,17 @@ def plot(data, countries, pop, **kwargs):
         with open("index.html.head", "r") as g:
             f.write(g.read())
         # table header
-        f.write("<table><tr><th>Land</th><th>Impfungen</th><th>Impfrate</th></tr>\n")
+        f.write("<table><tr><th>Land</th><th>Impfungen</th><th>Impfrate</th><th>Impfstoffe</th></tr>\n")
         # data
         for loc, tvac, rvac in vaccs:
-            f.write(f"<tr><td>{loc}</td><td>" + f"{tvac:,d}".replace(",",".") + f"</td><td>{rvac:3.3f}%</td></tr>\n".replace(".", ","))
+            line =  f"<tr><td>{loc}</td><td>" + \
+                    f"{tvac:,d}".replace(",",".") + \
+                    f"</td><td>{rvac:3.3f}%</td>".replace(".", ",")
+            if "vaccines" in metadata[loc]:
+                line += f"<td>{metadata[loc]['vaccines']}</td></tr>\n"
+            else:
+                line += f"<td>-</td></tr>\n"
+            f.write(line) 
         # table footer
         f.write("</table>\n")
         # site footer

coronavis.py

@@ -94,6 +94,7 @@ def get_data():
     metadata = {}
     with open(datafile, "r") as f:
         reader = csv.reader(f)
+
         for row in reader:
             if len(row) == 6:
                 date,location,new_cases,new_deaths,total_cases,total_deaths = row
@@ -105,6 +106,8 @@ def get_data():
                 iso_code,continent,location,date,total_cases,new_cases,new_cases_smoothed,total_deaths,new_deaths,new_deaths_smoothed,total_cases_per_million,new_cases_per_million,new_cases_smoothed_per_million,total_deaths_per_million,new_deaths_per_million,new_deaths_smoothed_per_million,reproduction_rate,icu_patients,icu_patients_per_million,hosp_patients,hosp_patients_per_million,weekly_icu_admissions,weekly_icu_admissions_per_million,weekly_hosp_admissions,weekly_hosp_admissions_per_million,new_tests,total_tests,total_tests_per_thousand,new_tests_per_thousand,new_tests_smoothed,new_tests_smoothed_per_thousand,positive_rate,tests_per_case,tests_units,total_vaccinations,total_vaccinations_per_hundred,stringency_index,population,population_density,median_age,aged_65_older,aged_70_older,gdp_per_capita,extreme_poverty,cardiovasc_death_rate,diabetes_prevalence,female_smokers,male_smokers,handwashing_facilities,hospital_beds_per_thousand,life_expectancy,human_development_index = row
             elif len(row) == 54:
                 iso_code, continent, location, date, total_cases, new_cases, new_cases_smoothed, total_deaths, new_deaths, new_deaths_smoothed, total_cases_per_million, new_cases_per_million, new_cases_smoothed_per_million, total_deaths_per_million, new_deaths_per_million, new_deaths_smoothed_per_million, reproduction_rate, icu_patients, icu_patients_per_million, hosp_patients, hosp_patients_per_million, weekly_icu_admissions, weekly_icu_admissions_per_million, weekly_hosp_admissions, weekly_hosp_admissions_per_million, new_tests, total_tests, total_tests_per_thousand, new_tests_per_thousand, new_tests_smoothed, new_tests_smoothed_per_thousand, positive_rate, tests_per_case, tests_units, total_vaccinations, new_vaccinations, total_vaccinations_per_hundred, new_vaccinations_per_million, stringency_index, population, population_density, median_age, aged_65_older, aged_70_older, gdp_per_capita, extreme_poverty, cardiovasc_death_rate, diabetes_prevalence, female_smokers, male_smokers, handwashing_facilities, hospital_beds_per_thousand, life_expectancy, human_development_index = row
+            elif len(row) == 55:
+                iso_code, continent, location, date, total_cases, new_cases, new_cases_smoothed, total_deaths, new_deaths, new_deaths_smoothed, total_cases_per_million, new_cases_per_million, new_cases_smoothed_per_million, total_deaths_per_million, new_deaths_per_million, new_deaths_smoothed_per_million, reproduction_rate, icu_patients, icu_patients_per_million, hosp_patients, hosp_patients_per_million, weekly_icu_admissions, weekly_icu_admissions_per_million, weekly_hosp_admissions, weekly_hosp_admissions_per_million, new_tests, total_tests, total_tests_per_thousand, new_tests_per_thousand, new_tests_smoothed, new_tests_smoothed_per_thousand, positive_rate, tests_per_case, tests_units, total_vaccinations, new_vaccinations, new_vaccinations_smoothed, total_vaccinations_per_hundred, new_vaccinations_smoothed_per_million, stringency_index, population, population_density, median_age, aged_65_older, aged_70_older, gdp_per_capita, extreme_poverty, cardiovasc_death_rate, diabetes_prevalence, female_smokers, male_smokers, handwashing_facilities, hospital_beds_per_thousand, life_expectancy, human_development_index = row
             else:
                 print(f"WARNING! Table format changed, length now {len(row)}, new header:\n{row})")
                 exit(1)
@@ -130,6 +133,7 @@ def get_data():
             total_tests = tofloat(total_tests)
             positive_rate = tofloat(positive_rate)
             tests_per_case = tofloat(tests_per_case)
+            new_vaccinations = tofloat(new_vaccinations)
             tests_units = tests_units
 
             if location not in data:
@@ -141,7 +145,8 @@ def get_data():
                     new_cases, new_deaths, total_cases, total_deaths, total_vaccinations,
                     stringency_index, reproduction_rate, icu_patients, hosp_patients,
                     weekly_icu_admissions, weekly_hosp_admissions, new_tests,
-                    total_tests, positive_rate, tests_per_case, tests_units,]
+                    total_tests, positive_rate, tests_per_case, tests_units,
+                    new_vaccinations,]
                     )
 
 
@@ -155,6 +160,21 @@ def get_data():
                     else:
                         if metadata[location][field] != row[n]:
                             print(f"{location}: {field} seems not to be a constant ({metadata[location][field]} vs {row[n]})")
+
+    ### End of csv reading loop
+
+# get data about vaccines
+    vaccinesurl = "https://github.com/owid/covid-19-data/raw/master/public/data/vaccinations/locations.csv"
+    vacraw = requests.get(vaccinesurl).content.decode("UTF8").split('\n')[1:-1]
+    vacreader = csv.reader(vacraw)
+    vaccines_country_dict = {}
+
+    for row in vacreader:
+        land = row[0]
+        vaccines = row[2]
+        vaccines_country_dict[land] = vaccines
+    del(vaccinesurl, vacraw, vacreader)
+
     # reorganize data
     data2 = {}
     for loc in data:
@@ -170,8 +190,9 @@ def get_data():
         positive_rate = []
         tests_per_case = []
         tests_units = []
+        new_vaccinations = []
         for entry in data[loc]:
-            t_, new_cases_, new_deaths_, total_cases_, total_deaths_, total_vaccinations_, stringency_index_, reproduction_rate_, icu_patients_, hosp_patients_, weekly_icu_admissions_, weekly_hosp_admissions_, new_tests_, total_tests_, positive_rate_, tests_per_case_, tests_units_ = entry
+            t_, new_cases_, new_deaths_, total_cases_, total_deaths_, total_vaccinations_, stringency_index_, reproduction_rate_, icu_patients_, hosp_patients_, weekly_icu_admissions_, weekly_hosp_admissions_, new_tests_, total_tests_, positive_rate_, tests_per_case_, tests_units_, new_vaccinations_ = entry
 
             time.append(t_)
             new_cases.append(toint(new_cases_))
@@ -189,7 +210,21 @@ def get_data():
             total_tests.append(toint(total_tests_))
             positive_rate.append(positive_rate_)
             tests_per_case.append(tests_per_case_)
+            new_vaccinations.append(toint(new_vaccinations_))
             tests_units.append(tests_units_)
+
+        ### data tweaking and fixing goes here
+
+        # fix vaccination data: not all countries report daily vaccinations
+        for n in range(1, len(total_vaccinations)):
+            if np.isnan(total_vaccinations[n]) and not np.isnan(total_vaccinations[n-1]):
+                total_vaccinations[n] = total_vaccinations[n-1]
+
+
+        ###
+
+
+        # collecting data
         data2[loc] = {'time': time,
                     'new_cases': new_cases,
                     'new_deaths': new_deaths,
@@ -207,11 +242,28 @@ def get_data():
                     'positive_rate': positive_rate,
                     'tests_per_case': tests_per_case,
                     'tests_units': tests_units,
+                    'new_vaccinations': new_vaccinations,
         }
+        # add vaccine info to metadata
+        if loc in vaccines_country_dict:
+            metadata[loc]['vaccines'] = vaccines_country_dict[loc]
+        # cast population to int
+        if loc != "International":
+            try: metadata[loc]['population'] = int(float(metadata[loc]['population']))
+            except: metadata[loc][loc]['population'] = np.nan
+
     return data2, metadata
 
 data, metadata = get_data()
 
+## dump data instead of plotting
+if False:
+    print("dumping data, no plotting")
+    import pickle
+    with open("data.dump", "wb") as f:
+        pickle.dump([data, metadata], f)
+    exit()
+
 for plot in plots:
     i = importlib.import_module(plot)
     i.plot(data, countries, pop, metadata=metadata)

country_details.py

@@ -9,6 +9,9 @@ import pickle
 import logging
 logging.getLogger().setLevel(logging.CRITICAL)
 import os
+import datetime
+import locale
+locale.setlocale(locale.LC_ALL, "de_DE.utf8")
 
 def plot(data, countries, pop, metadata, **kwargs):
     figsize = (10,5)
@@ -24,7 +27,31 @@ def plot(data, countries, pop, metadata, **kwargs):
         if not os.path.isdir(path):
             os.mkdir(path)
 
-        if not os.path.isfile(path+"/index.html") or True: # TODO enable html file generation
+        # is this country vaccinating?
+        is_vaccinating = True if data[loc]['total_vaccinations'][-1] > 0 else False
+        if is_vaccinating:
+            first_vac_report = np.argwhere(np.isnan(data[loc]['total_vaccinations']))[-1][0] + 1
+            vac_text = f"<tr><td>Impfstart</td><td>{data[loc]['time'][first_vac_report]}</td></tr>"
+            try:
+                vac_rate = data[loc]['total_vaccinations'][-1]/metadata[loc]['population']*100
+                vac_text+= f"<tr><td>Impfrate</td><td>{vac_rate:1.3f} %</td></tr>"
+            except:
+                pass
+            try:
+                immune_rate = (data[loc]['total_vaccinations'][-1] + data[loc]['total_cases'][-1])/metadata[loc]['population']*100
+                vac_text += f"<tr><td>Immunrate</td><td>{immune_rate:1.3f} %</td></tr>"
+            except:
+                pass
+            try:
+                vaccines = metadata[loc]['vaccines'] if 'vaccines' in metadata[loc] else "unknown"
+                vac_text += f"<tr><td>Impfstoffe</td><td>{vaccines}</td></tr>"
+            except:
+                pass
+        total_cases = data[loc]['total_cases'][-1]
+        total_deaths = data[loc]['total_deaths'][-1]
+        today = datetime.datetime.now().strftime("%d.%m.%Y") 
+
+        if True: #not os.path.isfile(path+"/index.html") or False: # TODO enable html file generation
             with open(path+"/index.html", "w") as f:
                 f.write(f"""
 <html>
@@ -42,6 +69,7 @@ def plot(data, countries, pop, metadata, **kwargs):
 <a href=https://dukun.de/corona>Zurück</a><br><br>
 
 <details open><summary><h4>Landesspezifische Kennzahlen</h4></summary>
+(Stand: {today})
 <table>
  <tr><td>ISO Code</td><td>{metadata[loc]["iso_code"]}</td></tr>
  <tr><td>Continent</td><td>{metadata[loc]["continent"]}</td></tr>
@@ -61,12 +89,18 @@ def plot(data, countries, pop, metadata, **kwargs):
  <tr><td>hospital beds per thousand inhabitants</td><td>{metadata[loc]["hospital_beds_per_thousand"]}</td></tr>
  <tr><td>life expectancy</td><td>{metadata[loc]["life_expectancy"]}</td></tr>
  <tr><td>human development index</td><td>{metadata[loc]["human_development_index"]}</td></tr>
+ <tr><td>Absolute bestätigte Infektionsfälle</td><td>{total_cases}</td></tr>
+ <tr><td>Absolute bestätigte Todesfälle</td><td>{total_deaths}</td></tr>
+{vac_text if is_vaccinating else ''}
 </table>
 </details>
 
 <details open><summary>Übersicht</summary><img src=https://dukun.de/corona/img/ac_all_{loc.replace(" ", "%20")}.png /></details>
 <details open><summary>Krankenhaussituation</summary><img src=https://dukun.de/corona/{path}/hospitals.png /></details>
-<details open><summary>Testsituation</summary><img src=https://dukun.de/corona/{path}/tests.png /></details>
+<details open><summary>Testsituation</summary><img src=https://dukun.de/corona/{path}/tests.png />
+<br>
+Testing dataset: Hasell, J., Mathieu, E., Beltekian, D. et al. A cross-country database of COVID-19 testing. Sci Data 7, 345 (2020). <a href=https://doi.org/10.1038/s41597-020-00688-8>DOI:10.1038/s41597-020-00688-8</a></details>
+<details open><summary>Impfsituation</summary><img src=https://dukun.de/corona/{path}/vaccinations.png /></details>
 
 <br><br>
 <a href=https://dukun.de/corona>Zurück</a><br>
@@ -81,6 +115,7 @@ Ein Infoservice von <a href=dukun.de>dukun.de</a>; Anregungen gern <a href="mail
         time = data[loc]['time']
         new_cases = data[loc]['new_cases']
         new_deaths = data[loc]['new_deaths']
+        new_vaccinations = data[loc]['new_vaccinations']
         total_cases = data[loc]['total_cases']
         total_deaths = data[loc]['total_deaths']
         total_vaccinations = data[loc]['total_vaccinations']
@@ -151,15 +186,19 @@ Ein Infoservice von <a href=dukun.de>dukun.de</a>; Anregungen gern <a href="mail
                 ttest_map = ~np.isnan(total_tests)
                 total_tests = np.array(total_tests)
                 new_tests = (total_tests[ttest_map][1:] - total_tests[ttest_map][:-1])/7.
-                ax1.plot(np.array(time)[ttest_map][1:], new_tests, color="blue", linestyle="-", linewidth=2, label="new tests")
+                try: ax1.plot(np.array(time)[ttest_map][1:], new_tests, color="blue", linestyle="-", linewidth=2, label="new tests")
+                except: pass
 
             elif not np.isnan(new_tests).all() :
                 ntest_map = ~np.isnan(new_tests)
-                ax1.plot(np.array(time)[ntest_map], np.array(new_tests)[ntest_map]/7, color="grey", linestyle="--", label="new tests")
+                try: ax1.plot(np.array(time)[ntest_map], np.array(new_tests)[ntest_map]/7, color="grey", linestyle="--", label="new tests")
-                ax1.plot(np.array(time)[ntest_map][3:-3], np.convolve(np.array(new_tests)[ntest_map], np.ones((7,))/7, mode="valid")/7, color="blue", linewidth=2, label="new tests 7day mean")
+                except: pass
+                try: ax1.plot(np.array(time)[ntest_map][3:-3], np.convolve(np.array(new_tests)[ntest_map], np.ones((7,))/7, mode="valid")/7, color="blue", linewidth=2, label="new tests 7day mean")
+                except: pass
             if not np.isnan(positive_rate).all():
                 prate_map = ~np.isnan(positive_rate)
-                ax2.plot(np.array(time)[prate_map], np.array(positive_rate)[prate_map]*100, color="black", linestyle="-", linewidth=2, label="positive rate (%)")
+                try: ax2.plot(np.array(time)[prate_map], np.array(positive_rate)[prate_map]*100, color="black", linestyle="-", linewidth=2, label="positive rate (%)")
+                except: pass
             
             ax1.set_ylabel(f"tests (unit: {testunit})")
             ax2.set_ylabel("positive rate")
@@ -170,8 +209,45 @@ Ein Infoservice von <a href=dukun.de>dukun.de</a>; Anregungen gern <a href="mail
             pp.savefig(path+"/tests.png")
             pp.close(fig)
 
+        ########### vaccine situation
+        if True:
+            fig, ax1 = pp.subplots(num=loc+"_vac", figsize=figsize)
+            ax2 = ax1.twinx()
+ 
+            if not np.isnan(new_vaccinations).all():
+                ax1.plot(np.array(time), new_vaccinations, color="grey", linestyle="--", linewidth=1, label="new vaccinations")
+            if not np.isnan(total_vaccinations).all():
+                ax2.plot(np.array(time), total_vaccinations, color="blue", linestyle="-", linewidth=1, label="total vaccinations")
+
+            immune_mask = ~np.isnan(total_vaccinations) & ~np.isnan(total_cases)
+            assert len(total_vaccinations) == len(total_cases)
+            total_immune = np.array(total_vaccinations) + np.array(total_cases)
+            ax2.plot(np.array(time)[immune_mask], total_immune[immune_mask], color="green", linestyle="-", linewidth=1, label="total immune")
+
+            ax1.set_ylabel(f"new vaccinations")
+            ax2.set_ylabel("total vaccinations")
+            fig.legend(frameon=False, loc="upper left", bbox_to_anchor=(0,1), bbox_transform=ax1.transAxes)
+            title = f"Vaccination situation in {loc}"
+            try:
+                title += f", Vaccines: {metadata[loc]['vaccines']}"
+            except: pass
+            try:
+                title += f", Immune rate: {immune_rate:1.2f}%"
+            except:
+                if loc == "Germany":
+                    print(f", Immune rate: {immune_rate:1.2f}%")
+
+                    exit()
+                pass
+            ax1.set_title(title)
+            
+            pp.text(0.002,0.005, f"plot generated {time_module.strftime('%Y-%m-%d %H:%M')}, CC-by-sa-nc, origin: dukun.de/corona, datasource: ourworldindata.org/coronavirus-source-data", color="dimgrey", fontsize=8, transform=fig.transFigure)
+            pp.savefig(path+"/vaccinations.png")
+            pp.close(fig)
+
+
 if __name__ == "__main__":
     import pickle
-    with open("20201221-data-metadata.dmp", "rb") as f:
+    with open("data.dump", "rb") as f:
         data, metadata = pickle.load(f)
     plot(data, [], {}, metadata=metadata)

index.html.head

@@ -24,8 +24,8 @@ bei 5, 50 und 500 festgesetzt, um ein bisschen die Schwere des Geschehens einsch
 Das wird aber ganz massiv durch die Testrate, Meldekette, politische Einflussnahme, betroffene Bevölkerungsschichten, betroffene Regionen, etc. beeinflusst und die praktische Bedeutung dieser Grenzwerte kann für
 die einzelnen Länder <b>sehr unterschiedlich</b> sein!
 <br><br>
-Die Daten stammen von <a href=https://ourworldindata.org/coronavirus-source-data>hier</a> und werden dort aus den WHO- und ECDC-Reports generiert.
+Die Daten stammen von <a href=https://ourworldindata.org/coronavirus-source-data>hier</a> und werden dort <a href=https://github.com/owid/covid-19-data/tree/master/public/data#data-on-covid-19-coronavirus-by-our-world-in-data>aus verschiedensten Quellen aggregiert</a>.
-Von den extrem reichhaltigen Daten dort verarbeite ich nur die Zahl der Neufälle.
+
 <br><br>
 Aktuelle Daten aus Deutschland mit vielen Hintergründen finden sich im <a href="https://www.rki.de/DE/Content/InfAZ/N/Neuartiges_Coronavirus/Situationsberichte/Gesamt.html">Lagebericht des RKI</a>.