various new features: new vaccination overview on country specific side, slightly changed text on index head, moved data correction to parser, made population an int, added the used vaccines to the vaccination table, and some bugfixing here and there

2021-01-21 18:42:48 +01:00
parent 3f0ce1b621
commit 71c4ef8065
4 changed files with 166 additions and 22 deletions
--- a/all_countries.py
+++ b/all_countries.py
@@ -23,7 +23,7 @@ corr = {"Chile": 10000,
        "Spain": 30000,
 }

-def plot(data, countries, pop, **kwargs):
+def plot(data, countries, pop, metadata={}, **kwargs):
    figsize = (10,5)
    
    vaccs = []
@@ -33,7 +33,7 @@ def plot(data, countries, pop, **kwargs):
            if loc == "International":
                continue
            name = basename+loc
-            time, new_cases, new_deaths, total_cases, total_deaths, total_vaccinations, stringency_index = data[loc]['time'], data[loc]['new_cases'], data[loc]['new_deaths'], data[loc]['total_cases'], data[loc]['total_deaths'], data[loc]['total_vaccinations'], data[loc]['stringency_index']
+            time, new_cases, new_deaths, total_cases, total_deaths, total_vaccinations, stringency_index, new_vaccinations = data[loc]['time'], data[loc]['new_cases'], data[loc]['new_deaths'], data[loc]['total_cases'], data[loc]['total_deaths'], data[loc]['total_vaccinations'], data[loc]['stringency_index'], data[loc]['new_vaccinations']
            
            fig, ax1 = pp.subplots(num=name, figsize=figsize)

@@ -49,16 +49,25 @@ def plot(data, countries, pop, **kwargs):
            ax1.plot(time[3:-3], np.convolve(new_cases, np.ones((7,))/7, mode="valid"), label="new cases 7day mean", color="orange", linestyle="-", linewidth=2)

            # plot vaccinations
-            if not np.isnan(total_vaccinations).all() :
-                print(f"{loc} has vaccines, adding to plot")
-                
-                # fix data: not all countries report daily
-                for n in range(1, len(total_vaccinations)):
-                    if np.isnan(total_vaccinations[n]) and not np.isnan(total_vaccinations[n-1]):
-                        total_vaccinations[n] = total_vaccinations[n-1]
+            if not np.isnan(total_vaccinations).all():
+                # notify of new vaccine programs
+                if np.isnan(total_vaccinations[-2]):
+                    print(f"{loc} starts vaccinating, adding to plot")

                ax2.plot(time, np.array(total_vaccinations), label=f"Total vaccinations", marker="", linestyle="-.", color="crimson")

+                if False:
+                    # plot detailed vaccination plot for all_countries.py
+                    rfig, rax = pp.subplots(1,1,num=f"{loc}_vacc")
+                    rax2 = rax.twinx()
+                    rax.plot(time, new_vaccinations, linestyle="--", color="green", label="new vac")
+                    rax2.plot(time, total_vaccinations, linestyle="-.", color="red", label="total vac")
+                    rax.set_ylabel("new vaccinatios")
+                    rax2.set_ylabel("total vaccinations")
+                    rax.legend(frameon=False, loc=2)
+
+                    rfig.savefig("img/"+f"{loc}".replace(" ", "_").replace("'", "").replace("/", "") + "/vaccs.png")
+
            # fix lower bound of plot
            for ax in (ax1, ax2):
                axis = ax.axis()
@@ -154,10 +163,17 @@ def plot(data, countries, pop, **kwargs):
        with open("index.html.head", "r") as g:
            f.write(g.read())
        # table header
-        f.write("<table><tr><th>Land</th><th>Impfungen</th><th>Impfrate</th></tr>\n")
+        f.write("<table><tr><th>Land</th><th>Impfungen</th><th>Impfrate</th><th>Impfstoffe</th></tr>\n")
        # data
        for loc, tvac, rvac in vaccs:
-            f.write(f"<tr><td>{loc}</td><td>" + f"{tvac:,d}".replace(",",".") + f"</td><td>{rvac:3.3f}%</td></tr>\n".replace(".", ","))
+            line =  f"<tr><td>{loc}</td><td>" + \
+                    f"{tvac:,d}".replace(",",".") + \
+                    f"</td><td>{rvac:3.3f}%</td>".replace(".", ",")
+            if "vaccines" in metadata[loc]:
+                line += f"<td>{metadata[loc]['vaccines']}</td></tr>\n"
+            else:
+                line += f"<td>-</td></tr>\n"
+            f.write(line) 
        # table footer
        f.write("</table>\n")
        # site footer
--- a/coronavis.py
+++ b/coronavis.py
@@ -94,6 +94,7 @@ def get_data():
    metadata = {}
    with open(datafile, "r") as f:
        reader = csv.reader(f)
+
        for row in reader:
            if len(row) == 6:
                date,location,new_cases,new_deaths,total_cases,total_deaths = row
@@ -105,6 +106,8 @@ def get_data():
                iso_code,continent,location,date,total_cases,new_cases,new_cases_smoothed,total_deaths,new_deaths,new_deaths_smoothed,total_cases_per_million,new_cases_per_million,new_cases_smoothed_per_million,total_deaths_per_million,new_deaths_per_million,new_deaths_smoothed_per_million,reproduction_rate,icu_patients,icu_patients_per_million,hosp_patients,hosp_patients_per_million,weekly_icu_admissions,weekly_icu_admissions_per_million,weekly_hosp_admissions,weekly_hosp_admissions_per_million,new_tests,total_tests,total_tests_per_thousand,new_tests_per_thousand,new_tests_smoothed,new_tests_smoothed_per_thousand,positive_rate,tests_per_case,tests_units,total_vaccinations,total_vaccinations_per_hundred,stringency_index,population,population_density,median_age,aged_65_older,aged_70_older,gdp_per_capita,extreme_poverty,cardiovasc_death_rate,diabetes_prevalence,female_smokers,male_smokers,handwashing_facilities,hospital_beds_per_thousand,life_expectancy,human_development_index = row
            elif len(row) == 54:
                iso_code, continent, location, date, total_cases, new_cases, new_cases_smoothed, total_deaths, new_deaths, new_deaths_smoothed, total_cases_per_million, new_cases_per_million, new_cases_smoothed_per_million, total_deaths_per_million, new_deaths_per_million, new_deaths_smoothed_per_million, reproduction_rate, icu_patients, icu_patients_per_million, hosp_patients, hosp_patients_per_million, weekly_icu_admissions, weekly_icu_admissions_per_million, weekly_hosp_admissions, weekly_hosp_admissions_per_million, new_tests, total_tests, total_tests_per_thousand, new_tests_per_thousand, new_tests_smoothed, new_tests_smoothed_per_thousand, positive_rate, tests_per_case, tests_units, total_vaccinations, new_vaccinations, total_vaccinations_per_hundred, new_vaccinations_per_million, stringency_index, population, population_density, median_age, aged_65_older, aged_70_older, gdp_per_capita, extreme_poverty, cardiovasc_death_rate, diabetes_prevalence, female_smokers, male_smokers, handwashing_facilities, hospital_beds_per_thousand, life_expectancy, human_development_index = row
+            elif len(row) == 55:
+                iso_code, continent, location, date, total_cases, new_cases, new_cases_smoothed, total_deaths, new_deaths, new_deaths_smoothed, total_cases_per_million, new_cases_per_million, new_cases_smoothed_per_million, total_deaths_per_million, new_deaths_per_million, new_deaths_smoothed_per_million, reproduction_rate, icu_patients, icu_patients_per_million, hosp_patients, hosp_patients_per_million, weekly_icu_admissions, weekly_icu_admissions_per_million, weekly_hosp_admissions, weekly_hosp_admissions_per_million, new_tests, total_tests, total_tests_per_thousand, new_tests_per_thousand, new_tests_smoothed, new_tests_smoothed_per_thousand, positive_rate, tests_per_case, tests_units, total_vaccinations, new_vaccinations, new_vaccinations_smoothed, total_vaccinations_per_hundred, new_vaccinations_smoothed_per_million, stringency_index, population, population_density, median_age, aged_65_older, aged_70_older, gdp_per_capita, extreme_poverty, cardiovasc_death_rate, diabetes_prevalence, female_smokers, male_smokers, handwashing_facilities, hospital_beds_per_thousand, life_expectancy, human_development_index = row
            else:
                print(f"WARNING! Table format changed, length now {len(row)}, new header:\n{row})")
                exit(1)
@@ -130,6 +133,7 @@ def get_data():
            total_tests = tofloat(total_tests)
            positive_rate = tofloat(positive_rate)
            tests_per_case = tofloat(tests_per_case)
+            new_vaccinations = tofloat(new_vaccinations)
            tests_units = tests_units

            if location not in data:
@@ -141,7 +145,8 @@ def get_data():
                    new_cases, new_deaths, total_cases, total_deaths, total_vaccinations,
                    stringency_index, reproduction_rate, icu_patients, hosp_patients,
                    weekly_icu_admissions, weekly_hosp_admissions, new_tests,
-                    total_tests, positive_rate, tests_per_case, tests_units,]
+                    total_tests, positive_rate, tests_per_case, tests_units,
+                    new_vaccinations,]
                    )


@@ -155,6 +160,21 @@ def get_data():
                    else:
                        if metadata[location][field] != row[n]:
                            print(f"{location}: {field} seems not to be a constant ({metadata[location][field]} vs {row[n]})")
+
+    ### End of csv reading loop
+
+# get data about vaccines
+    vaccinesurl = "https://github.com/owid/covid-19-data/raw/master/public/data/vaccinations/locations.csv"
+    vacraw = requests.get(vaccinesurl).content.decode("UTF8").split('\n')[1:-1]
+    vacreader = csv.reader(vacraw)
+    vaccines_country_dict = {}
+
+    for row in vacreader:
+        land = row[0]
+        vaccines = row[2]
+        vaccines_country_dict[land] = vaccines
+    del(vaccinesurl, vacraw, vacreader)
+
    # reorganize data
    data2 = {}
    for loc in data:
@@ -170,8 +190,9 @@ def get_data():
        positive_rate = []
        tests_per_case = []
        tests_units = []
+        new_vaccinations = []
        for entry in data[loc]:
-            t_, new_cases_, new_deaths_, total_cases_, total_deaths_, total_vaccinations_, stringency_index_, reproduction_rate_, icu_patients_, hosp_patients_, weekly_icu_admissions_, weekly_hosp_admissions_, new_tests_, total_tests_, positive_rate_, tests_per_case_, tests_units_ = entry
+            t_, new_cases_, new_deaths_, total_cases_, total_deaths_, total_vaccinations_, stringency_index_, reproduction_rate_, icu_patients_, hosp_patients_, weekly_icu_admissions_, weekly_hosp_admissions_, new_tests_, total_tests_, positive_rate_, tests_per_case_, tests_units_, new_vaccinations_ = entry

            time.append(t_)
            new_cases.append(toint(new_cases_))
@@ -189,7 +210,21 @@ def get_data():
            total_tests.append(toint(total_tests_))
            positive_rate.append(positive_rate_)
            tests_per_case.append(tests_per_case_)
+            new_vaccinations.append(toint(new_vaccinations_))
            tests_units.append(tests_units_)
+
+        ### data tweaking and fixing goes here
+
+        # fix vaccination data: not all countries report daily vaccinations
+        for n in range(1, len(total_vaccinations)):
+            if np.isnan(total_vaccinations[n]) and not np.isnan(total_vaccinations[n-1]):
+                total_vaccinations[n] = total_vaccinations[n-1]
+
+
+        ###
+
+
+        # collecting data
        data2[loc] = {'time': time,
                    'new_cases': new_cases,
                    'new_deaths': new_deaths,
@@ -207,11 +242,28 @@ def get_data():
                    'positive_rate': positive_rate,
                    'tests_per_case': tests_per_case,
                    'tests_units': tests_units,
+                    'new_vaccinations': new_vaccinations,
        }
+        # add vaccine info to metadata
+        if loc in vaccines_country_dict:
+            metadata[loc]['vaccines'] = vaccines_country_dict[loc]
+        # cast population to int
+        if loc != "International":
+            try: metadata[loc]['population'] = int(float(metadata[loc]['population']))
+            except: metadata[loc][loc]['population'] = np.nan
+
    return data2, metadata

 data, metadata = get_data()

+## dump data instead of plotting
+if False:
+    print("dumping data, no plotting")
+    import pickle
+    with open("data.dump", "wb") as f:
+        pickle.dump([data, metadata], f)
+    exit()
+
 for plot in plots:
    i = importlib.import_module(plot)
    i.plot(data, countries, pop, metadata=metadata)
--- a/country_details.py
+++ b/country_details.py
@@ -9,6 +9,9 @@ import pickle
 import logging
 logging.getLogger().setLevel(logging.CRITICAL)
 import os
+import datetime
+import locale
+locale.setlocale(locale.LC_ALL, "de_DE.utf8")

 def plot(data, countries, pop, metadata, **kwargs):
    figsize = (10,5)
@@ -24,7 +27,31 @@ def plot(data, countries, pop, metadata, **kwargs):
        if not os.path.isdir(path):
            os.mkdir(path)

-        if not os.path.isfile(path+"/index.html") or True: # TODO enable html file generation
+        # is this country vaccinating?
+        is_vaccinating = True if data[loc]['total_vaccinations'][-1] > 0 else False
+        if is_vaccinating:
+            first_vac_report = np.argwhere(np.isnan(data[loc]['total_vaccinations']))[-1][0] + 1
+            vac_text = f"<tr><td>Impfstart</td><td>{data[loc]['time'][first_vac_report]}</td></tr>"
+            try:
+                vac_rate = data[loc]['total_vaccinations'][-1]/metadata[loc]['population']*100
+                vac_text+= f"<tr><td>Impfrate</td><td>{vac_rate:1.3f} %</td></tr>"
+            except:
+                pass
+            try:
+                immune_rate = (data[loc]['total_vaccinations'][-1] + data[loc]['total_cases'][-1])/metadata[loc]['population']*100
+                vac_text += f"<tr><td>Immunrate</td><td>{immune_rate:1.3f} %</td></tr>"
+            except:
+                pass
+            try:
+                vaccines = metadata[loc]['vaccines'] if 'vaccines' in metadata[loc] else "unknown"
+                vac_text += f"<tr><td>Impfstoffe</td><td>{vaccines}</td></tr>"
+            except:
+                pass
+        total_cases = data[loc]['total_cases'][-1]
+        total_deaths = data[loc]['total_deaths'][-1]
+        today = datetime.datetime.now().strftime("%d.%m.%Y") 
+
+        if True: #not os.path.isfile(path+"/index.html") or False: # TODO enable html file generation
            with open(path+"/index.html", "w") as f:
                f.write(f"""
 <html>
@@ -42,6 +69,7 @@ def plot(data, countries, pop, metadata, **kwargs):
 <a href=https://dukun.de/corona>Zurück</a><br><br>

 <details open><summary><h4>Landesspezifische Kennzahlen</h4></summary>
+(Stand: {today})
 <table>
 <tr><td>ISO Code</td><td>{metadata[loc]["iso_code"]}</td></tr>
 <tr><td>Continent</td><td>{metadata[loc]["continent"]}</td></tr>
@@ -61,12 +89,18 @@ def plot(data, countries, pop, metadata, **kwargs):
 <tr><td>hospital beds per thousand inhabitants</td><td>{metadata[loc]["hospital_beds_per_thousand"]}</td></tr>
 <tr><td>life expectancy</td><td>{metadata[loc]["life_expectancy"]}</td></tr>
 <tr><td>human development index</td><td>{metadata[loc]["human_development_index"]}</td></tr>
+ <tr><td>Absolute bestätigte Infektionsfälle</td><td>{total_cases}</td></tr>
+ <tr><td>Absolute bestätigte Todesfälle</td><td>{total_deaths}</td></tr>
+{vac_text if is_vaccinating else ''}
 </table>
 </details>

 <details open><summary>Übersicht</summary><img src=https://dukun.de/corona/img/ac_all_{loc.replace(" ", "%20")}.png /></details>
 <details open><summary>Krankenhaussituation</summary><img src=https://dukun.de/corona/{path}/hospitals.png /></details>
-<details open><summary>Testsituation</summary><img src=https://dukun.de/corona/{path}/tests.png /></details>
+<details open><summary>Testsituation</summary><img src=https://dukun.de/corona/{path}/tests.png />
+<br>
+Testing dataset: Hasell, J., Mathieu, E., Beltekian, D. et al. A cross-country database of COVID-19 testing. Sci Data 7, 345 (2020). <a href=https://doi.org/10.1038/s41597-020-00688-8>DOI:10.1038/s41597-020-00688-8</a></details>
+<details open><summary>Impfsituation</summary><img src=https://dukun.de/corona/{path}/vaccinations.png /></details>

 <br><br>
 <a href=https://dukun.de/corona>Zurück</a><br>
@@ -81,6 +115,7 @@ Ein Infoservice von <a href=dukun.de>dukun.de</a>; Anregungen gern <a href="mail
        time = data[loc]['time']
        new_cases = data[loc]['new_cases']
        new_deaths = data[loc]['new_deaths']
+        new_vaccinations = data[loc]['new_vaccinations']
        total_cases = data[loc]['total_cases']
        total_deaths = data[loc]['total_deaths']
        total_vaccinations = data[loc]['total_vaccinations']
@@ -151,15 +186,19 @@ Ein Infoservice von <a href=dukun.de>dukun.de</a>; Anregungen gern <a href="mail
                ttest_map = ~np.isnan(total_tests)
                total_tests = np.array(total_tests)
                new_tests = (total_tests[ttest_map][1:] - total_tests[ttest_map][:-1])/7.
-                ax1.plot(np.array(time)[ttest_map][1:], new_tests, color="blue", linestyle="-", linewidth=2, label="new tests")
+                try: ax1.plot(np.array(time)[ttest_map][1:], new_tests, color="blue", linestyle="-", linewidth=2, label="new tests")
+                except: pass

            elif not np.isnan(new_tests).all() :
                ntest_map = ~np.isnan(new_tests)
-                ax1.plot(np.array(time)[ntest_map], np.array(new_tests)[ntest_map]/7, color="grey", linestyle="--", label="new tests")
-                ax1.plot(np.array(time)[ntest_map][3:-3], np.convolve(np.array(new_tests)[ntest_map], np.ones((7,))/7, mode="valid")/7, color="blue", linewidth=2, label="new tests 7day mean")
+                try: ax1.plot(np.array(time)[ntest_map], np.array(new_tests)[ntest_map]/7, color="grey", linestyle="--", label="new tests")
+                except: pass
+                try: ax1.plot(np.array(time)[ntest_map][3:-3], np.convolve(np.array(new_tests)[ntest_map], np.ones((7,))/7, mode="valid")/7, color="blue", linewidth=2, label="new tests 7day mean")
+                except: pass
            if not np.isnan(positive_rate).all():
                prate_map = ~np.isnan(positive_rate)
-                ax2.plot(np.array(time)[prate_map], np.array(positive_rate)[prate_map]*100, color="black", linestyle="-", linewidth=2, label="positive rate (%)")
+                try: ax2.plot(np.array(time)[prate_map], np.array(positive_rate)[prate_map]*100, color="black", linestyle="-", linewidth=2, label="positive rate (%)")
+                except: pass
            
            ax1.set_ylabel(f"tests (unit: {testunit})")
            ax2.set_ylabel("positive rate")
@@ -170,8 +209,45 @@ Ein Infoservice von <a href=dukun.de>dukun.de</a>; Anregungen gern <a href="mail
            pp.savefig(path+"/tests.png")
            pp.close(fig)

+        ########### vaccine situation
+        if True:
+            fig, ax1 = pp.subplots(num=loc+"_vac", figsize=figsize)
+            ax2 = ax1.twinx()
+ 
+            if not np.isnan(new_vaccinations).all():
+                ax1.plot(np.array(time), new_vaccinations, color="grey", linestyle="--", linewidth=1, label="new vaccinations")
+            if not np.isnan(total_vaccinations).all():
+                ax2.plot(np.array(time), total_vaccinations, color="blue", linestyle="-", linewidth=1, label="total vaccinations")
+
+            immune_mask = ~np.isnan(total_vaccinations) & ~np.isnan(total_cases)
+            assert len(total_vaccinations) == len(total_cases)
+            total_immune = np.array(total_vaccinations) + np.array(total_cases)
+            ax2.plot(np.array(time)[immune_mask], total_immune[immune_mask], color="green", linestyle="-", linewidth=1, label="total immune")
+
+            ax1.set_ylabel(f"new vaccinations")
+            ax2.set_ylabel("total vaccinations")
+            fig.legend(frameon=False, loc="upper left", bbox_to_anchor=(0,1), bbox_transform=ax1.transAxes)
+            title = f"Vaccination situation in {loc}"
+            try:
+                title += f", Vaccines: {metadata[loc]['vaccines']}"
+            except: pass
+            try:
+                title += f", Immune rate: {immune_rate:1.2f}%"
+            except:
+                if loc == "Germany":
+                    print(f", Immune rate: {immune_rate:1.2f}%")
+
+                    exit()
+                pass
+            ax1.set_title(title)
+            
+            pp.text(0.002,0.005, f"plot generated {time_module.strftime('%Y-%m-%d %H:%M')}, CC-by-sa-nc, origin: dukun.de/corona, datasource: ourworldindata.org/coronavirus-source-data", color="dimgrey", fontsize=8, transform=fig.transFigure)
+            pp.savefig(path+"/vaccinations.png")
+            pp.close(fig)
+
+
 if __name__ == "__main__":
    import pickle
-    with open("20201221-data-metadata.dmp", "rb") as f:
+    with open("data.dump", "rb") as f:
        data, metadata = pickle.load(f)
    plot(data, [], {}, metadata=metadata)
--- a/index.html.head
+++ b/index.html.head
@@ -24,8 +24,8 @@ bei 5, 50 und 500 festgesetzt, um ein bisschen die Schwere des Geschehens einsch
 Das wird aber ganz massiv durch die Testrate, Meldekette, politische Einflussnahme, betroffene Bevölkerungsschichten, betroffene Regionen, etc. beeinflusst und die praktische Bedeutung dieser Grenzwerte kann für
 die einzelnen Länder <b>sehr unterschiedlich</b> sein!
 <br><br>
-Die Daten stammen von <a href=https://ourworldindata.org/coronavirus-source-data>hier</a> und werden dort aus den WHO- und ECDC-Reports generiert.
-Von den extrem reichhaltigen Daten dort verarbeite ich nur die Zahl der Neufälle.
+Die Daten stammen von <a href=https://ourworldindata.org/coronavirus-source-data>hier</a> und werden dort <a href=https://github.com/owid/covid-19-data/tree/master/public/data#data-on-covid-19-coronavirus-by-our-world-in-data>aus verschiedensten Quellen aggregiert</a>.
+
 <br><br>
 Aktuelle Daten aus Deutschland mit vielen Hintergründen finden sich im <a href="https://www.rki.de/DE/Content/InfAZ/N/Neuartiges_Coronavirus/Situationsberichte/Gesamt.html">Lagebericht des RKI</a>.