Automation and configuration improvements

CHANGES:

1. Lancaster CC risk report integrated into pipeline;
2. DSTL Scotland estimates adaptor code incorporated;
3. Update to covid19uk v0.8.0-alpha.5

covid_pipeline/ruffus_pipeline.py

@@ -29,6 +29,7 @@ from covid_pipeline.tasks import (
     summary_longformat,
     crystalcast_output,
     summary_dha,
+    lancs_risk_report,
 )
 
 __all__ = ["run_pipeline"]
@@ -268,4 +269,14 @@ def run_pipeline(global_config, results_directory, cli_options):
             url="https://fhm-chicas-storage.lancs.ac.uk/bayesstm/latest/",
         )
 
+    # Lancashire CC report
+    @rf.transform(
+        input=summary_geopackage,
+        filter=rf.formatter(),
+        output=wd("lancs_risk_report.xlsx"),
+        )
+    def lancs(input, output):
+        lancs_risk_report(input, output, pipeline_meta['created_at'])
+
+        
     rf.cmdline.run(cli_options)

covid_pipeline/tasks/__init__.py

@@ -10,6 +10,7 @@ from covid_pipeline.tasks.summary_longformat import summary_longformat
 import covid_pipeline.tasks.summarize as summarize
 from covid_pipeline.tasks.crystalcast_output import crystalcast_output
 from covid_pipeline.tasks.summary_dha import summary_dha
+from covid_pipeline.tasks.lancs_risk_report import report_lancs as lancs_risk_report
 
 __all__ = [
     "overall_rt",
@@ -20,4 +21,5 @@ __all__ = [
     "summarize",
     "crystalcast_output",
     "summary_dha",
+    "lancs_risk_report",
 ]

england_pillar1_config.yaml

@@ -34,7 +34,10 @@ Mcmc:
   num_burst_samples: 200  # Number of MCMC samples per burst
   thin: 20  # Thin MCMC samples every `thin` iterations
   num_adaptation_iterations: 1000
-
+  initial_rates:
+    ei: 0.5
+    ir: 0.37
+    
 ThinPosterior:  # Post-process further chain thinning HDF5 -> .pkl.
   start: -10000
   end: 

pipeline.sge

@@ -64,7 +64,10 @@ module add cuda/11.0
 
 export XLA_FLAGS="--xla_gpu_cuda_data_dir=$CUDA_HOME"
 
-
+DHA_DIR="s3://bayesstm/latest"
+BASENAME=`basename ${arg_staging_dir}`
+BUCKET_DIR=${arg_results_dir}/${BASENAME}
+GEOGRAPHY=`echo ${BASENAME} | cut -d"_" -f2`
 DATEHIGH=`date -d "${arg_date}" +%F`
 DATELOW=`date -d "${arg_date} - 84 days" +%F`
 
@@ -76,25 +79,35 @@ poetry run python -m covid_pipeline.pipeline \
       -v ||
     error_exit "Pipeline job failed"
 
-# Push results to bucket
-BASENAME=`basename ${arg_staging_dir}`
-BUCKET_DIR=${arg_results_dir}/${BASENAME}
-DHA_DIR="bayesstm/latest"
 
+# Adapt Scotland data format
+if [ $GEOGRAPHY = uk ]; then
+    echo "Adapting Scotland data"
+    module add R
+    Rscript R/Clean_Lancaster_data.R \
+	    ${arg_staging_dir}/summary_longformat.xlsx \
+	    ${arg_staging_dir}/scotland_output.xlsx || error_exit "Scotland data failed"
+    module remove R
+fi
+
+
+# Push results to buckets
+echo "Push to ${BUCKET_DIR}"
 module add aws-cli
 aws --endpoint-url https://fhm-chicas-storage.lancs.ac.uk s3 \
     --profile covid-pipeline-data \
     cp --recursive ${arg_staging_dir} ${BUCKET_DIR}
 
 # Push dha to bayesstm bucket
-aws --endpoint-url https://fhm-chicas-storage.lancs.ac.uk s3 \
-    --profile bayesstm \
-    cp --recursive ${arg_staging_dir}/dha ${DHA_DIR}
-
+if [ $GEOGRAPHY = uk ]; then
+    echo "Push to $DHA_DIR"
+    aws --endpoint-url https://fhm-chicas-storage.lancs.ac.uk s3 \
+	--profile bayesstm \
+	cp --recursive "${arg_staging_dir}/dha" ${DHA_DIR}
+fi
 
 # Trigger report
 REPORTBRANCH=master
-GEOGRAPHY=`echo ${BASENAME} | cut -d"_" -f2`
 echo "Reporting for geography '$GEOGRAPHY'"
 if [ $GEOGRAPHY = england1 ]; then
     REPORTBRANCH=england1
@@ -106,4 +119,7 @@ curl -X POST \
      -F "variables[S3TARGET]=${BUCKET_DIR}" \
      https://fhm-chicas-code.lancs.ac.uk/api/v4/projects/163/trigger/pipeline
 
+echo "Pipeline succeeded, writing notification file to $global_storage/covid_notify/${GEOGRAPHY}_complete.notify"
+echo ${arg_staging_dir} > ${global_storage}/covid_notify/${GEOGRAPHY}_complete.notify
+
 exit 0

pyproject.toml

@@ -16,7 +16,7 @@ descartes = "^1.1.0"
 enum34 = "1.1.8"
 mapclassify = "^2.4.2"
 dhaconfig = {git = "https://gitlab.com/achale/dhaconfig.git", tag="v0.0.2"}
-covid19uk = {git = "https://gitlab.com/chicas-covid19/covid19uk.git", tag="v0.8.0-alpha.4"}
+covid19uk = {git = "https://gitlab.com/chicas-covid19/covid19uk.git", tag="v0.8.0-alpha.5"}
 
 [tool.poetry.dev-dependencies]
 jedi = "^0.17.2"

uk_config.yaml

@@ -36,7 +36,9 @@ Mcmc:
   num_burst_samples: 200  # Number of MCMC samples per burst
   thin: 20  # Thin MCMC samples every `thin` iterations
   num_adaptation_iterations: 1000
-
+  initial_rates:
+    ei: 0.5
+    ir: 0.37
 
 ThinPosterior:  # Post-process further chain thinning HDF5 -> .pkl.
   start: -10000