Initial commit

LICENSE

+Copyright (c) 2020 Chris Jewell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

+# ddgrab README
+
+(c) Chris Jewell <c.jewell@lancaster.ac.uk> 2020
+
+## Introduction
+
+`ddgrab`, or "Dated Data Grab", provides a short script that connects to a SFTP server and copies the most recent ISO8601 date-named folder to a local machine.For example, if the remote server has directory structure
+
+```
+/-
+ |--some_folder
+  |--2020-02-01
+  |--2020-10-02
+```
+then the script will download the folder named `2020-10-02` given a
+`remote_folder` setting of `some_folder`. 
+
+It is designed to be run periodically, and uses filename/filesize combinations
+to determine if any new or modified files are available.  If they are, then
+the whole data is re-downloaded and replaces that on the local system.
+
+## Installation
+
+Installation is via `pip`, for example:
+
+```
+pip install <path to ddgrab>
+```
+
+Note that this project uses the new PEP517 standard,
+and supports `pip>=10.0.0` only.
+
+## Usage
+ 
+There are two ways to use `ddgrab`, from within a Python script, and from the command line.
+
+### Within Python
+
+To poll a server `ftp.somewhere.com` for a remote folder `some_folder`, and
+download to a local folder `local_dir/myprefix_2020-10-02`, use
+
+```
+from phecovid import get_data
+
+server = "ftp.somewhere.com"
+remote_folder = "some_folder"
+secrets = {"username": "foo",
+           "password": "bar"}
+localdir = "/home/user/data"
+prefix = "myprefix_"
+
+get_data(server, remote_folder, secrets, localdir, prefix)
+```
+
+If called repeatedly with the same local directory and prefix,
+the data will only be downloaded if something in the remote directory
+changes.
+
+
+### From the command line
+
+`ddgrab` installs a script which can be used to run it from the command
+line.  For command line use, place your secrets (username/password) in 
+a yaml file like so:
+
+```
+username: fred
+password: pa$$w0rd
+```
+
+Assuming we've saved our secrets in a file `secrets.yaml`, use the command
+line utility like so:
+
+```
+$ ddgrab --server sftp.domain.com --remote_folder foo \
+   --secrets secrets.yaml --localdir downloads --folder-prefix data_
+```

ddgrab/__init__.py

+__version__ = "0.1.0"
+
+from ddgrab.main import get_data

ddgrab/main.py

+"""Grabs DSTL COVID-19 data from sftp service"""
+
+import argparse
+import re
+import sys
+import shutil
+import tempfile
+from pathlib import Path
+from warnings import warn
+
+import yaml
+import pysftp as ftp
+
+
+class TransferInProgressError(Exception):
+    pass
+
+
+def get_most_recent(date_dirs):
+    """For a list of directories, some of which have an ISO8601
+    date as a name, return the directory with the most recent name.
+    """
+    dm = re.compile("^[0-9]{4}-[0-9]{2}-[0-9]{2}")
+    dirs = [elem for elem in date_dirs if dm.match(elem)]
+    dirs.sort(reverse=True)
+    return dirs[0]
+
+
+def maybe_clean_path(path):
+    if path.exists():
+        shutil.rmtree(path)
+
+
+# A very simple directory locking protocol
+#  to prevent multiple writes to the same data.
+# We avoid POSIX flock() because it's not guaranteed
+#  to work over CIFS/NFS.
+def lockfile(path):
+    return path / Path("_transfer_in_progress")
+
+
+def agnostic_lock(path):
+    try:
+        lockfile(path).touch(exist_ok=False)
+    except FileExistsError:
+        raise TransferInProgressError(f"Another transfer to `{path}` is in progress.")
+
+
+def agnostic_unlock(path):
+    if lockfile(path).exists():
+        lockfile(path).unlink()
+
+
+def get_data(server, remote_folder, secrets, localdir, dir_prefix=""):
+    """Searches for the latest ISO8601 date-named folder in
+    `server:remote_folder`, with authentication
+    `secrets`, and downloads to `localdir`, prepending the new local
+     directory with `dir_prefix`.
+
+    Downloads are only performed if the local directory
+    contains all the files in the remote directory, and those files all
+    have the same sizes.  This is conservative, but we cannot perform
+    remote checksums, so we don't trust any of the local data if anything
+    changes on the remote side.
+
+    :param server: the sftp server
+    :param remote_folder: the folder on the SFTP server in which to look for
+                          ISO8601-named folders to download.
+    :param secrets: a Python dictionary with keys `username` and `password` containing
+                     SFTP credentials.
+    :param localdir: the local download directory
+    :param dir_prefix: a prefix to prepend to the dated folder name. e.g. `2020-10-02` becomes
+                        `dir_prefix2020-10-02`.
+
+    :returns: None
+    :raises: `TransferInProgressError` if the directory is locked, implying
+              another process is writing to it.
+    """
+    with ftp.Connection(
+        server, username=secrets["username"], password=secrets["password"]
+    ) as con:
+        with con.cd(remote_folder):
+            dirs = con.listdir()
+
+            # Take date directories only
+            most_recent = get_most_recent(dirs)
+            prefixed_most_recent = dir_prefix + most_recent
+
+            # Create local dir if needed
+            target_path = Path(localdir) / Path(prefixed_most_recent)
+            target_path.mkdir(parents=True, exist_ok=True)
+
+            # Get filenames and filesize on both sides
+            try:
+                local_files = set(
+                    [(f.name, f.stat().st_size) for f in target_path.glob("*")]
+                )
+                with con.cd(most_recent):
+                    remote_files = set(
+                        [(f.filename, f.st_size) for f in con.listdir_attr()]
+                    )
+
+                # Download only if remote files are not all present in local dir
+                if not remote_files.issubset(local_files):
+                    print(f"Downloading to {target_path}", file=sys.stderr)
+                    agnostic_lock(target_path)
+                    con.get_d(most_recent, target_path, preserve_mtime=True)
+                    agnostic_unlock(target_path)
+                else:
+                    print(f"Download up to date. Exit.", file=sys.stderr)
+            except TransferInProgressError as e:
+                raise e
+            except Exception as e:  # Clean up lock, unless TransferInProgressError
+                agnostic_unlock(target_path)
+                raise e
+
+
+def run():
+    """Type `dstlcovid -h` for further information"""
+
+    parser = argparse.ArgumentParser("Download data")
+    parser.add_argument("--server", type=str, help="The SFTP server")
+    parser.add_argument(
+        "--remote_folder", type=str, help="The remote folder to connect to"
+    )
+    parser.add_argument("--secrets", type=str, help="Secrets file")
+    parser.add_argument("--localdir", type=str, help="Local download directory")
+    parser.add_argument("--dir-prefix", type=str, default="", help="Prefix folder")
+    args = parser.parse_args()
+
+    with open(args.secrets, "r") as f:
+        secrets = yaml.load(f)
+
+    try:
+        get_data(
+            server=args.server,
+            remote_folder=args.remote_folder,
+            secrets=secrets,
+            localdir=args.localdir,
+            dir_prefix=args.dir_prefix,
+        )
+    except Exception as e:
+        warn(f"Error in data download. {e}")
+        sys.exit(1)
+
+    sys.exit(0)

pyproject.toml

+[tool.poetry]
+name = "ddgrab"
+version = "0.1.0"
+description = ""
+authors = ["Chris Jewell <c.jewell@lancaster.ac.uk>"]
+
+[tool.poetry.dependencies]
+python = "^3.7"
+PyYAML = "^5.3.1"
+pysftp = "^0.2.9"
+
+[tool.poetry.dev-dependencies]
+pytest = "^5.2"
+
+[tool.poetry.scripts]
+ddgrab = 'ddgrab.main:run'
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

tests/test_ddgrab.py

+from ddgrab import __version__
+
+
+def test_version():
+    assert __version__ == "0.1.0"