Commit 23cde9b8 authored by Chris Jewell's avatar Chris Jewell
Browse files

Corrected ruffus match term for aws upload

parent 16bad2b7
......@@ -64,9 +64,7 @@ if __name__ == "__main__":
data_args.add_argument(
"--pillar", type=str, help="Pillar", choices=["both", "1", "2"]
)
data_args.add_argument(
"--aws", action='store_true', help="Push to AWS"
)
data_args.add_argument("--aws", action="store_true", help="Push to AWS")
cli_options = argparser.parse_args()
global_config = _import_global_config(cli_options.config)
......@@ -105,13 +103,3 @@ if __name__ == "__main__":
]
run_pipeline(global_config, cli_options.results_directory, cli_options)
if cli_options.aws is True:
bucket_name = global_config['AWSS3']['bucket']
obj_name = os.path.split(cli_options.results_directory)[1]
obj_path = f"{bucket_name}/{obj_name}"
s3 = s3fs.S3FileSystem(profile=global_config["AWSS3"]["profile"])
if not s3.exists(obj_path):
s3.put(cli_options.results_directory, obj_path, recursive=True)
else:
warnings.warn(f"Path '{obj_path}' already exists, not uploading.")
"""Represents the analytic pipeline as a ruffus chain"""
import os
import warnings
import yaml
from datetime import datetime
from uuid import uuid1
import json
import s3fs
import netCDF4 as nc
import pandas as pd
import ruffus as rf
......@@ -228,4 +230,28 @@ def run_pipeline(global_config, results_directory, cli_options):
wd("summary_longformat.xlsx"),
)(summary_longformat)
# Copy results to AWS
@rf.active_if(cli_options.aws)
@rf.transform(
input=[
process_data,
mcmc,
insample7,
insample14,
medium_term,
reproduction_number,
],
filter=rf.formatter(),
output="{subdir[0][0]}/{basename[0]}{ext[0]}",
extras=[global_config["AWSS3"]],
)
def upload_to_aws(input_file, output_file, config):
obj_path = f"{config['bucket']}/{output_file}"
s3 = s3fs.S3FileSystem(profile=config["profile"])
if not s3.exists(obj_path):
s3.put(input_file, obj_path)
else:
warnings.warn(f"Path '{obj_path}' already exists, not uploading.")
rf.cmdline.run(cli_options)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment