feat: support s3 better

removed pipenv, lint clean up

4 files changed, 225 insertions, 272 deletions

diff --git a/Pipfile b/Pipfile
deleted file mode 100644
index 4194aee..0000000
--- a/Pipfile
+++ /dev/null
@@ -1,16 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[dev-packages]
-ipython = "*"
-ipdb = "*"
-"autopep8" = "*"
-
-[packages]
-sh = "*"
-restic = {path = "."}
-
-[requires]
-python_version = "3.6"
diff --git a/Pipfile.lock b/Pipfile.lock
deleted file mode 100644
index 20667e9..0000000
--- a/Pipfile.lock
+++ /dev/null
@@ -1,155 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "ded3c2dd868d603f64fab2bdffb55751b970ba0a2ab481597f9cb52e4bdd0b1e"
-        },
-        "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3.6"
-        },
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "restic": {
-            "path": "."
-        },
-        "sh": {
-            "hashes": [
-                "sha256:ae3258c5249493cebe73cb4e18253a41ed69262484bad36fdb3efcb8ad8870bb",
-                "sha256:b52bf5833ed01c7b5c5fb73a7f71b3d98d48e9b9b8764236237bdc7ecae850fc"
-            ],
-            "index": "pypi",
-            "version": "==1.12.14"
-        }
-    },
-    "develop": {
-        "autopep8": {
-            "hashes": [
-                "sha256:1b8d42ebba751a91090d3adb5c06840b1151d71ed43e1c7a9ed6911bfe8ebe6c"
-            ],
-            "index": "pypi",
-            "version": "==1.4.2"
-        },
-        "backcall": {
-            "hashes": [
-                "sha256:38ecd85be2c1e78f77fd91700c76e14667dc21e2713b63876c0eb901196e01e4",
-                "sha256:bbbf4b1e5cd2bdb08f915895b51081c041bac22394fdfcfdfbe9f14b77c08bf2"
-            ],
-            "version": "==0.1.0"
-        },
-        "decorator": {
-            "hashes": [
-                "sha256:2c51dff8ef3c447388fe5e4453d24a2bf128d3a4c32af3fabef1f01c6851ab82",
-                "sha256:c39efa13fbdeb4506c476c9b3babf6a718da943dab7811c206005a4a956c080c"
-            ],
-            "version": "==4.3.0"
-        },
-        "ipdb": {
-            "hashes": [
-                "sha256:7081c65ed7bfe7737f83fa4213ca8afd9617b42ff6b3f1daf9a3419839a2a00a"
-            ],
-            "index": "pypi",
-            "version": "==0.11"
-        },
-        "ipython": {
-            "hashes": [
-                "sha256:a5781d6934a3341a1f9acb4ea5acdc7ea0a0855e689dbe755d070ca51e995435",
-                "sha256:b10a7ddd03657c761fc503495bc36471c8158e3fc948573fb9fe82a7029d8efd"
-            ],
-            "index": "pypi",
-            "version": "==7.1.1"
-        },
-        "ipython-genutils": {
-            "hashes": [
-                "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8",
-                "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"
-            ],
-            "version": "==0.2.0"
-        },
-        "jedi": {
-            "hashes": [
-                "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7",
-                "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148"
-            ],
-            "version": "==0.13.1"
-        },
-        "parso": {
-            "hashes": [
-                "sha256:35704a43a3c113cce4de228ddb39aab374b8004f4f2407d070b6a2ca784ce8a2",
-                "sha256:895c63e93b94ac1e1690f5fdd40b65f07c8171e3e53cbd7793b5b96c0e0a7f24"
-            ],
-            "version": "==0.3.1"
-        },
-        "pexpect": {
-            "hashes": [
-                "sha256:2a8e88259839571d1251d278476f3eec5db26deb73a70be5ed5dc5435e418aba",
-                "sha256:3fbd41d4caf27fa4a377bfd16fef87271099463e6fa73e92a52f92dfee5d425b"
-            ],
-            "markers": "sys_platform != 'win32'",
-            "version": "==4.6.0"
-        },
-        "pickleshare": {
-            "hashes": [
-                "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca",
-                "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"
-            ],
-            "version": "==0.7.5"
-        },
-        "prompt-toolkit": {
-            "hashes": [
-                "sha256:c1d6aff5252ab2ef391c2fe498ed8c088066f66bc64a8d5c095bbf795d9fec34",
-                "sha256:d4c47f79b635a0e70b84fdb97ebd9a274203706b1ee5ed44c10da62755cf3ec9",
-                "sha256:fd17048d8335c1e6d5ee403c3569953ba3eb8555d710bfc548faf0712666ea39"
-            ],
-            "version": "==2.0.7"
-        },
-        "ptyprocess": {
-            "hashes": [
-                "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0",
-                "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f"
-            ],
-            "version": "==0.6.0"
-        },
-        "pycodestyle": {
-            "hashes": [
-                "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83",
-                "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a"
-            ],
-            "version": "==2.4.0"
-        },
-        "pygments": {
-            "hashes": [
-                "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d",
-                "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc"
-            ],
-            "version": "==2.2.0"
-        },
-        "six": {
-            "hashes": [
-                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
-                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
-            ],
-            "version": "==1.11.0"
-        },
-        "traitlets": {
-            "hashes": [
-                "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835",
-                "sha256:c6cb5e6f57c5a9bdaa40fa71ce7b4af30298fbab9ece9815b5d995ab6217c7d9"
-            ],
-            "version": "==4.3.2"
-        },
-        "wcwidth": {
-            "hashes": [
-                "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
-                "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
-            ],
-            "version": "==0.1.7"
-        }
-    }
-}
diff --git a/restic.ini b/restic.ini
index d9bc4e0..28f606c 100644
--- a/restic.ini
+++ b/restic.ini
@@ -1,3 +1,3 @@
 [restic]
 exclude=~/.local/etc/restic-wrapper/exclude.conf
-repo_uri=sftp:vito.bing:/home/srv/grumps-repo
+repo_uri=s3:https://s3.bing.c-137.space/backups
diff --git a/restic.py b/restic.py
index 95be705..0d8a0b0 100644
--- a/restic.py
+++ b/restic.py
@@ -7,131 +7,255 @@ All rights reserved.
 restic-py is simple wrapper to restic binary for backups and monitoring
 """
 import os
-import tempfile
 import configparser
-import glob
 import argparse
 import logging
 import sys
 import time
-import io
-
-import sh
+import socket
+import getpass
+import subprocess
+import pathlib
+import collections
+import shlex
 
 # logging
 LOGLEVELS = {'DEBUG': logging.DEBUG,
              'INFO': logging.INFO,
              'WARNING': logging.WARNING,
              'ERROR': logging.ERROR}
-
 HOME_DIR = os.path.expanduser('~')
-DEFAULT_CONFIG = '{}/.local/etc/restic-wrapper/restic.ini'.format(HOME_DIR)
-CONFIG = os.environ.get('RESTIC_CONFIG', DEFAULT_CONFIG)
+DEFAULTZ = {
+    'namespace': f'{getpass.getuser()}-{socket.gethostname()}',
+    'exclude_file': pathlib.Path(f'{HOME_DIR}/.local/etc/restic-wrapper/exclude.conf'),
+    'password_file': pathlib.Path(f'{HOME_DIR}/.ssh/resticpass'),
+}
+ResticCmdConfig = collections.namedtuple(
+    'ResticCmdConfig', ['exec_bin', 'base_args', 'exclude']
+)
 
-# TODO should be a configurable location
-logging.basicConfig(filename='{}/.local/logs/restic.log'.format(HOME_DIR),
-                    level=LOGLEVELS[os.getenv('RESTIC_LOGLEVEL', 'INFO')])
+# Log Config
+log_args = {'filename': f'{HOME_DIR}/.local/logs/restic.log'}
+if sys.stdin.isatty():
+    log_file = sys.stdout
+    log_args = {'stream': sys.stdout}
+log_args['level'] = LOGLEVELS[os.getenv('RESTIC_LOGLEVEL', 'INFO')]
+logging.basicConfig(**log_args)
 log = logging.getLogger(__name__)
+# End Log Config
+
+
+def prepare_args():
+    """prepare arg parser"""
+    parser = argparse.ArgumentParser(description='wrapper to restic')
+    parser.add_argument('-c', '--config',
+                        default=f'{HOME_DIR}/.local/etc/restic-wrapper/restic.ini',
+                        type=pathlib.Path)
+    parser.add_argument('-e', '--exclude-file',
+                        type=pathlib.Path)
+    parser.add_argument('-n', '--namespace')
+    parser.add_argument('-u', '--restic-uri')
+    parser.add_argument('-k', '--password-file')
+    subparsers = parser.add_subparsers(title='wrapper commands',
+                                       description='valid subcommands',
+                                       required=True)
+    backup_parser = subparsers.add_parser('backup')
+    backup_parser.set_defaults(func=run_backup)
+    backup_parser.add_argument('path',
+                               nargs='?',
+                               default=pathlib.Path('.'),
+                               help='path to backup',
+                               type=pathlib.Path)
+    prune_parser = subparsers.add_parser('prune')
+    prune_parser.set_defaults(func=run_prune)
+    check_parser = subparsers.add_parser('check')
+    check_parser.set_defaults(func=run_check)
+    watch_parser = subparsers.add_parser('watch')
+    watch_parser.set_defaults(func=run_watch)
+    pre_operation_check_parser = subparsers.add_parser('validate-config')
+    pre_operation_check_parser.set_defaults(func=run_pre_operation_check)
+    args = parser.parse_args()
+    # load ini file
+    cfg = configparser.ConfigParser()
+    cfg.read(args.config)
+    # build a dictionary of from restic ini section
+    config_file_args = dict(cfg.items('restic'))
+    # build dictionary of the args
+    command_line_args = {k: v for k, v in vars(args).items() if v is not None}
+    combined = collections.ChainMap(command_line_args,
+                                    os.environ,
+                                    config_file_args,
+                                    DEFAULTZ)
+    return combined
+
+
+# subprocess utils
+def run(cmd, *cmd_args, stdin=None, cstdout=False, cstderr=False, cwd=None):
+    """runs a given command"""
+    runcmd = [cmd, *cmd_args]
+    runcmd = [str(s) for s in runcmd]
+    runcmd = shlex.join(runcmd)
+    stderr_pipe = subprocess.PIPE if cstderr else None
+    stdout_pipe = subprocess.PIPE if cstdout else None
+    _r = subprocess.run(shlex.split(runcmd), stdout=stdout_pipe, stderr=stderr_pipe,
+                        input=stdin, cwd=cwd, check=False)
+    stdout = _r.stdout
+    stderr =_r.stderr
+    if not stdout:
+        stdout = b''
+    if not stderr:
+        stderr = b''
+    exit_clean = _r.returncode == 0
+    return exit_clean, stdout, stderr
+# end suprocess utils
+
+
+def resolve_restic():
+    """resolve path torestic binary"""
+    env_paths = os.environ.get('PATH', '').split(os.pathsep)
+    for env_path in env_paths:
+        exec_bin = pathlib.Path(env_path) / 'restic'
+        if not exec_bin.is_file():
+            continue
+        return exec_bin
+    return ''
 
-# cmd line parser
-parser = argparse.ArgumentParser(description='wrapper to restic')
-parser.add_argument('cmd',
-                    type=str,
-                    choices=('backup', 'prune', 'check', 'watch'),
-                    help='run backup')
-
-# config
-cfg = configparser.ConfigParser()
-cfg.read(CONFIG)
-
-PASSWORD_FILE = '{}/.ssh/resticpass'.format(HOME_DIR)
-exclude_file = os.path.expanduser(cfg['restic']['exclude'])
-restic_uri = cfg['restic']['repo_uri']
-BACKUP_CMD = sh.restic.bake('backup',
-                            '--password-file', PASSWORD_FILE,
-                            '--exclude-file', exclude_file,
-                            '--exclude-caches',
-                            '--repo', restic_uri)
-PRUNE_CMD = sh.restic.bake('forget',
-                           '--password-file', PASSWORD_FILE,
-                           '--repo', restic_uri,
-                           '--keep-daily', 7,
-                           '--keep-weekly', 5,
-                           '--keep-monthly', 12,
-                           '--keep-yearly', 75)
-CHECK_CMD = sh.restic.bake('check',
-                           '--password-file', PASSWORD_FILE,
-                           '--repo', restic_uri)
-
-
-def _check_auth():
+# pylint: disable=unused-argument
+def run_pre_operation_check(rconf, args):
+    """validate authentication, repo access, and configuration"""
+    log.info('validating config')
+    if not check_auth():
+        return False
+    status, _, stderr = run(
+        rconf.exec_bin,
+        'cat', 'config',
+        *rconf.base_args,
+        cstderr=1,
+        cstdout=1)
+    if not status:
+        log.error('pre repo check failed: %s', stderr)
+        return False
+    return True
+
+
+def check_auth():
+    """check authentication with restic backend"""
     try:
-        os.environ['SSH_AUTH_SOCK']
+        # pylint: disable=pointless-statement
+        os.environ['AWS_SECRET_ACCESS_KEY']
+        # pylint: disable=pointless-statement
+        os.environ['AWS_ACCESS_KEY_ID']
     except KeyError:
-        log.error('ssh agent not running')
-
-
-def run_backup():
-    directories = []
-    for chunk in glob.iglob('{}/*'.format(HOME_DIR)):
-        if os.path.isdir(chunk):
-            try:
-                out = BACKUP_CMD(chunk)
-                directories.append(chunk)
-                log.info('{} {}'.format('restic-run', chunk))
-            except sh.ErrorReturnCode_1 as e:
-                log.error('{} {} {}'.format('restic-run', chunk, e))
-    # backs up non chunked files
-    with tempfile.NamedTemporaryFile(delete=True) as tmp:
-        tmp.write('\n'.join(directories).encode())
-        out = BACKUP_CMD('--exclude-file', tmp.name, HOME_DIR)
-        log.info('{} {}'.format('restic-run', HOME_DIR))
-
-
-def run_prune():
+        log.error('AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID are required.')
+        return False
+    return True
+
+
+def run_backup(rconf, argz):
+    """backup a given directory"""
     try:
-        out = PRUNE_CMD()
-    except sh.ErrorReturnCode_1 as e:
-        log.error('{} {}'.format('restic-prune', e))
+        log.info('restic-backup start')
+        run(rconf.exec_bin,
+            'backup',
+            '--exclude-caches',
+            *rconf.exclude,
+            argz['path'],
+            *rconf.base_args)
+    except Exception as error:
+        log.error('restic-run failed %s', error)
+        raise error
+    log.info('restic-backup finished')
+    return True
 
+def run_prune(rconf, argz):
+    """run prune task"""
+    try:
+        run(rconf.exec_bin,
+            'prune',
+            *rconf.base_args)
+    # pylint: disable=broad-exception-caught
+    except Exception as error:
+        log.error('resitc-prune %s', error)
+        return False
+    return True
 
-def run_check():
+def run_forget(rconf, argz):
+    """forget backups that are no longer needed"""
     try:
-        out = CHECK_CMD()
-    except sh.ErrorReturnCode_1 as e:
-        log.error('{} {}'.format('restic-check', e))
-
-
-def follow(logfile, pattern):
-    # catch up
-    for line in logfile.readlines():
-        if pattern in line:
-            yield line
-    logfile.seek(0, 2)
-    while True:
-        line = logfile.readline()
-        if not line:
-            time.sleep(0.1)
-            continue
-        if pattern in line:
-            yield line
+        status, _, stderr = run(rconf.exec_bin,
+            'forget',
+            '--keep-daily', 7,
+            '--keep-weekly', 5,
+            '--keep-monthly', 12,
+            '--keep-yearly', 75,
+            *rconf.base_args)
+        if not status:
+            log.error('restic-prune %s', stderr)
+            return False
+    # pylint: disable=broad-exception-caught
+    except Exception as error:
+        log.error('retsic-prune %s', error)
+        return False
+    return True
 
+def run_check(rconf, args):
+    """restic check command"""
+    try:
+        status, _, stderr = run(rconf.exec_bin,
+                                'check',
+                                *rconf.base_args)
+        if not status:
+            log.error('restic-check %s', stderr)
+            return False
+    # pylint: disable=broad-exception-caught
+    except Exception as error:
+        log.error('restic-check %s', error)
+        return False
+    return True
 
-def run_watch():
-    logfile = open("restic.log","r")
-    loglines = follow(logfile, 'ERROR')
-    # TODO notification
-    for line in loglines:
-        print(line)
 
+# def follow(logfile, pattern):
+#     # catch up
+#     for line in logfile.readlines():
+#         if pattern in line:
+#             yield line
+#     logfile.seek(0, 2)
+#     while True:
+#         line = logfile.readline()
+#         if not line:
+#             time.sleep(0.1)
+#             continue
+#         if pattern in line:
+#             yield line
+#
+#
+# def run_watch(args):
+#     logfile = open("restic.log", "r")
+#     loglines = follow(logfile, 'ERROR')
+#     for line in loglines:
+#         print(line)
+#
 
 def main():
-    _check_auth()
-    cmd_arg = parser.parse_args()
-    cmd = 'run_{}'.format(cmd_arg.cmd)
+    """main"""
+    argz = prepare_args()
+    # ResticCmdConfig
+    restic_config = ResticCmdConfig(
+        resolve_restic(),
+        [
+            '--password-file', argz['password_file'],
+            '--repo', f'{argz["repo"]}/{argz["namespace"]}',
+        ],
+        [
+            '--exclude-file', argz['exclude_file'],
+        ]
+    )
     try:
-        run = globals()[cmd]
-    except KeyError:
-        log.error('command {} not found'.format(cmd))
+        success = argz['func'](restic_config, argz)
+    # pylint: disable=broad-exception-caught
+    except Exception as error:
+        log.error('error running %s', error)
+        sys.exit(1)
+    if not success:
         sys.exit(1)
-    run()
+    sys.exit(1)