Commit 0a665e9c authored by Chris Jewell's avatar Chris Jewell
Browse files

Automation and configuration improvements

CHANGES:

1. Lancaster CC risk report integrated into pipeline;
2. DSTL Scotland estimates adaptor code incorporated;
3. Update to covid19uk v0.8.0-alpha.5
parent b3635b3b
......@@ -29,6 +29,7 @@ from covid_pipeline.tasks import (
summary_longformat,
crystalcast_output,
summary_dha,
lancs_risk_report,
)
__all__ = ["run_pipeline"]
......@@ -268,4 +269,14 @@ def run_pipeline(global_config, results_directory, cli_options):
url="https://fhm-chicas-storage.lancs.ac.uk/bayesstm/latest/",
)
# Lancashire CC report
@rf.transform(
input=summary_geopackage,
filter=rf.formatter(),
output=wd("lancs_risk_report.xlsx"),
)
def lancs(input, output):
lancs_risk_report(input, output, pipeline_meta['created_at'])
rf.cmdline.run(cli_options)
......@@ -10,6 +10,7 @@ from covid_pipeline.tasks.summary_longformat import summary_longformat
import covid_pipeline.tasks.summarize as summarize
from covid_pipeline.tasks.crystalcast_output import crystalcast_output
from covid_pipeline.tasks.summary_dha import summary_dha
from covid_pipeline.tasks.lancs_risk_report import report_lancs as lancs_risk_report
__all__ = [
"overall_rt",
......@@ -20,4 +21,5 @@ __all__ = [
"summarize",
"crystalcast_output",
"summary_dha",
"lancs_risk_report",
]
......@@ -34,7 +34,10 @@ Mcmc:
num_burst_samples: 200 # Number of MCMC samples per burst
thin: 20 # Thin MCMC samples every `thin` iterations
num_adaptation_iterations: 1000
initial_rates:
ei: 0.5
ir: 0.37
ThinPosterior: # Post-process further chain thinning HDF5 -> .pkl.
start: -10000
end:
......
......@@ -64,7 +64,10 @@ module add cuda/11.0
export XLA_FLAGS="--xla_gpu_cuda_data_dir=$CUDA_HOME"
DHA_DIR="s3://bayesstm/latest"
BASENAME=`basename ${arg_staging_dir}`
BUCKET_DIR=${arg_results_dir}/${BASENAME}
GEOGRAPHY=`echo ${BASENAME} | cut -d"_" -f2`
DATEHIGH=`date -d "${arg_date}" +%F`
DATELOW=`date -d "${arg_date} - 84 days" +%F`
......@@ -76,25 +79,35 @@ poetry run python -m covid_pipeline.pipeline \
-v ||
error_exit "Pipeline job failed"
# Push results to bucket
BASENAME=`basename ${arg_staging_dir}`
BUCKET_DIR=${arg_results_dir}/${BASENAME}
DHA_DIR="bayesstm/latest"
# Adapt Scotland data format
if [ $GEOGRAPHY = uk ]; then
echo "Adapting Scotland data"
module add R
Rscript R/Clean_Lancaster_data.R \
${arg_staging_dir}/summary_longformat.xlsx \
${arg_staging_dir}/scotland_output.xlsx || error_exit "Scotland data failed"
module remove R
fi
# Push results to buckets
echo "Push to ${BUCKET_DIR}"
module add aws-cli
aws --endpoint-url https://fhm-chicas-storage.lancs.ac.uk s3 \
--profile covid-pipeline-data \
cp --recursive ${arg_staging_dir} ${BUCKET_DIR}
# Push dha to bayesstm bucket
aws --endpoint-url https://fhm-chicas-storage.lancs.ac.uk s3 \
--profile bayesstm \
cp --recursive ${arg_staging_dir}/dha ${DHA_DIR}
if [ $GEOGRAPHY = uk ]; then
echo "Push to $DHA_DIR"
aws --endpoint-url https://fhm-chicas-storage.lancs.ac.uk s3 \
--profile bayesstm \
cp --recursive "${arg_staging_dir}/dha" ${DHA_DIR}
fi
# Trigger report
REPORTBRANCH=master
GEOGRAPHY=`echo ${BASENAME} | cut -d"_" -f2`
echo "Reporting for geography '$GEOGRAPHY'"
if [ $GEOGRAPHY = england1 ]; then
REPORTBRANCH=england1
......@@ -106,4 +119,7 @@ curl -X POST \
-F "variables[S3TARGET]=${BUCKET_DIR}" \
https://fhm-chicas-code.lancs.ac.uk/api/v4/projects/163/trigger/pipeline
echo "Pipeline succeeded, writing notification file to $global_storage/covid_notify/${GEOGRAPHY}_complete.notify"
echo ${arg_staging_dir} > ${global_storage}/covid_notify/${GEOGRAPHY}_complete.notify
exit 0
......@@ -16,7 +16,7 @@ descartes = "^1.1.0"
enum34 = "1.1.8"
mapclassify = "^2.4.2"
dhaconfig = {git = "https://gitlab.com/achale/dhaconfig.git", tag="v0.0.2"}
covid19uk = {git = "https://gitlab.com/chicas-covid19/covid19uk.git", tag="v0.8.0-alpha.4"}
covid19uk = {git = "https://gitlab.com/chicas-covid19/covid19uk.git", tag="v0.8.0-alpha.5"}
[tool.poetry.dev-dependencies]
jedi = "^0.17.2"
......
......@@ -36,7 +36,9 @@ Mcmc:
num_burst_samples: 200 # Number of MCMC samples per burst
thin: 20 # Thin MCMC samples every `thin` iterations
num_adaptation_iterations: 1000
initial_rates:
ei: 0.5
ir: 0.37
ThinPosterior: # Post-process further chain thinning HDF5 -> .pkl.
start: -10000
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment