clean up stuffrefactor-drop-sh-support-s3

2 files changed, 225 insertions, 101 deletions

diff --git a/restic.ini b/restic.ini
index d9bc4e0..28f606c 100644
--- a/restic.ini
+++ b/restic.ini
@@ -1,3 +1,3 @@
 [restic]
 exclude=~/.local/etc/restic-wrapper/exclude.conf
-repo_uri=sftp:vito.bing:/home/srv/grumps-repo
+repo_uri=s3:https://s3.bing.c-137.space/backups
diff --git a/restic.py b/restic.py
index 95be705..0d8a0b0 100644
--- a/restic.py
+++ b/restic.py
@@ -7,131 +7,255 @@ All rights reserved.
 restic-py is simple wrapper to restic binary for backups and monitoring
 """
 import os
-import tempfile
 import configparser
-import glob
 import argparse
 import logging
 import sys
 import time
-import io
-
-import sh
+import socket
+import getpass
+import subprocess
+import pathlib
+import collections
+import shlex
 
 # logging
 LOGLEVELS = {'DEBUG': logging.DEBUG,
              'INFO': logging.INFO,
              'WARNING': logging.WARNING,
              'ERROR': logging.ERROR}
-
 HOME_DIR = os.path.expanduser('~')
-DEFAULT_CONFIG = '{}/.local/etc/restic-wrapper/restic.ini'.format(HOME_DIR)
-CONFIG = os.environ.get('RESTIC_CONFIG', DEFAULT_CONFIG)
+DEFAULTZ = {
+    'namespace': f'{getpass.getuser()}-{socket.gethostname()}',
+    'exclude_file': pathlib.Path(f'{HOME_DIR}/.local/etc/restic-wrapper/exclude.conf'),
+    'password_file': pathlib.Path(f'{HOME_DIR}/.ssh/resticpass'),
+}
+ResticCmdConfig = collections.namedtuple(
+    'ResticCmdConfig', ['exec_bin', 'base_args', 'exclude']
+)
 
-# TODO should be a configurable location
-logging.basicConfig(filename='{}/.local/logs/restic.log'.format(HOME_DIR),
-                    level=LOGLEVELS[os.getenv('RESTIC_LOGLEVEL', 'INFO')])
+# Log Config
+log_args = {'filename': f'{HOME_DIR}/.local/logs/restic.log'}
+if sys.stdin.isatty():
+    log_file = sys.stdout
+    log_args = {'stream': sys.stdout}
+log_args['level'] = LOGLEVELS[os.getenv('RESTIC_LOGLEVEL', 'INFO')]
+logging.basicConfig(**log_args)
 log = logging.getLogger(__name__)
+# End Log Config
+
+
+def prepare_args():
+    """prepare arg parser"""
+    parser = argparse.ArgumentParser(description='wrapper to restic')
+    parser.add_argument('-c', '--config',
+                        default=f'{HOME_DIR}/.local/etc/restic-wrapper/restic.ini',
+                        type=pathlib.Path)
+    parser.add_argument('-e', '--exclude-file',
+                        type=pathlib.Path)
+    parser.add_argument('-n', '--namespace')
+    parser.add_argument('-u', '--restic-uri')
+    parser.add_argument('-k', '--password-file')
+    subparsers = parser.add_subparsers(title='wrapper commands',
+                                       description='valid subcommands',
+                                       required=True)
+    backup_parser = subparsers.add_parser('backup')
+    backup_parser.set_defaults(func=run_backup)
+    backup_parser.add_argument('path',
+                               nargs='?',
+                               default=pathlib.Path('.'),
+                               help='path to backup',
+                               type=pathlib.Path)
+    prune_parser = subparsers.add_parser('prune')
+    prune_parser.set_defaults(func=run_prune)
+    check_parser = subparsers.add_parser('check')
+    check_parser.set_defaults(func=run_check)
+    watch_parser = subparsers.add_parser('watch')
+    watch_parser.set_defaults(func=run_watch)
+    pre_operation_check_parser = subparsers.add_parser('validate-config')
+    pre_operation_check_parser.set_defaults(func=run_pre_operation_check)
+    args = parser.parse_args()
+    # load ini file
+    cfg = configparser.ConfigParser()
+    cfg.read(args.config)
+    # build a dictionary of from restic ini section
+    config_file_args = dict(cfg.items('restic'))
+    # build dictionary of the args
+    command_line_args = {k: v for k, v in vars(args).items() if v is not None}
+    combined = collections.ChainMap(command_line_args,
+                                    os.environ,
+                                    config_file_args,
+                                    DEFAULTZ)
+    return combined
+
+
+# subprocess utils
+def run(cmd, *cmd_args, stdin=None, cstdout=False, cstderr=False, cwd=None):
+    """runs a given command"""
+    runcmd = [cmd, *cmd_args]
+    runcmd = [str(s) for s in runcmd]
+    runcmd = shlex.join(runcmd)
+    stderr_pipe = subprocess.PIPE if cstderr else None
+    stdout_pipe = subprocess.PIPE if cstdout else None
+    _r = subprocess.run(shlex.split(runcmd), stdout=stdout_pipe, stderr=stderr_pipe,
+                        input=stdin, cwd=cwd, check=False)
+    stdout = _r.stdout
+    stderr =_r.stderr
+    if not stdout:
+        stdout = b''
+    if not stderr:
+        stderr = b''
+    exit_clean = _r.returncode == 0
+    return exit_clean, stdout, stderr
+# end suprocess utils
+
+
+def resolve_restic():
+    """resolve path torestic binary"""
+    env_paths = os.environ.get('PATH', '').split(os.pathsep)
+    for env_path in env_paths:
+        exec_bin = pathlib.Path(env_path) / 'restic'
+        if not exec_bin.is_file():
+            continue
+        return exec_bin
+    return ''
 
-# cmd line parser
-parser = argparse.ArgumentParser(description='wrapper to restic')
-parser.add_argument('cmd',
-                    type=str,
-                    choices=('backup', 'prune', 'check', 'watch'),
-                    help='run backup')
-
-# config
-cfg = configparser.ConfigParser()
-cfg.read(CONFIG)
-
-PASSWORD_FILE = '{}/.ssh/resticpass'.format(HOME_DIR)
-exclude_file = os.path.expanduser(cfg['restic']['exclude'])
-restic_uri = cfg['restic']['repo_uri']
-BACKUP_CMD = sh.restic.bake('backup',
-                            '--password-file', PASSWORD_FILE,
-                            '--exclude-file', exclude_file,
-                            '--exclude-caches',
-                            '--repo', restic_uri)
-PRUNE_CMD = sh.restic.bake('forget',
-                           '--password-file', PASSWORD_FILE,
-                           '--repo', restic_uri,
-                           '--keep-daily', 7,
-                           '--keep-weekly', 5,
-                           '--keep-monthly', 12,
-                           '--keep-yearly', 75)
-CHECK_CMD = sh.restic.bake('check',
-                           '--password-file', PASSWORD_FILE,
-                           '--repo', restic_uri)
-
-
-def _check_auth():
+# pylint: disable=unused-argument
+def run_pre_operation_check(rconf, args):
+    """validate authentication, repo access, and configuration"""
+    log.info('validating config')
+    if not check_auth():
+        return False
+    status, _, stderr = run(
+        rconf.exec_bin,
+        'cat', 'config',
+        *rconf.base_args,
+        cstderr=1,
+        cstdout=1)
+    if not status:
+        log.error('pre repo check failed: %s', stderr)
+        return False
+    return True
+
+
+def check_auth():
+    """check authentication with restic backend"""
     try:
-        os.environ['SSH_AUTH_SOCK']
+        # pylint: disable=pointless-statement
+        os.environ['AWS_SECRET_ACCESS_KEY']
+        # pylint: disable=pointless-statement
+        os.environ['AWS_ACCESS_KEY_ID']
     except KeyError:
-        log.error('ssh agent not running')
-
-
-def run_backup():
-    directories = []
-    for chunk in glob.iglob('{}/*'.format(HOME_DIR)):
-        if os.path.isdir(chunk):
-            try:
-                out = BACKUP_CMD(chunk)
-                directories.append(chunk)
-                log.info('{} {}'.format('restic-run', chunk))
-            except sh.ErrorReturnCode_1 as e:
-                log.error('{} {} {}'.format('restic-run', chunk, e))
-    # backs up non chunked files
-    with tempfile.NamedTemporaryFile(delete=True) as tmp:
-        tmp.write('\n'.join(directories).encode())
-        out = BACKUP_CMD('--exclude-file', tmp.name, HOME_DIR)
-        log.info('{} {}'.format('restic-run', HOME_DIR))
-
-
-def run_prune():
+        log.error('AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID are required.')
+        return False
+    return True
+
+
+def run_backup(rconf, argz):
+    """backup a given directory"""
     try:
-        out = PRUNE_CMD()
-    except sh.ErrorReturnCode_1 as e:
-        log.error('{} {}'.format('restic-prune', e))
+        log.info('restic-backup start')
+        run(rconf.exec_bin,
+            'backup',
+            '--exclude-caches',
+            *rconf.exclude,
+            argz['path'],
+            *rconf.base_args)
+    except Exception as error:
+        log.error('restic-run failed %s', error)
+        raise error
+    log.info('restic-backup finished')
+    return True
 
+def run_prune(rconf, argz):
+    """run prune task"""
+    try:
+        run(rconf.exec_bin,
+            'prune',
+            *rconf.base_args)
+    # pylint: disable=broad-exception-caught
+    except Exception as error:
+        log.error('resitc-prune %s', error)
+        return False
+    return True
 
-def run_check():
+def run_forget(rconf, argz):
+    """forget backups that are no longer needed"""
     try:
-        out = CHECK_CMD()
-    except sh.ErrorReturnCode_1 as e:
-        log.error('{} {}'.format('restic-check', e))
-
-
-def follow(logfile, pattern):
-    # catch up
-    for line in logfile.readlines():
-        if pattern in line:
-            yield line
-    logfile.seek(0, 2)
-    while True:
-        line = logfile.readline()
-        if not line:
-            time.sleep(0.1)
-            continue
-        if pattern in line:
-            yield line
+        status, _, stderr = run(rconf.exec_bin,
+            'forget',
+            '--keep-daily', 7,
+            '--keep-weekly', 5,
+            '--keep-monthly', 12,
+            '--keep-yearly', 75,
+            *rconf.base_args)
+        if not status:
+            log.error('restic-prune %s', stderr)
+            return False
+    # pylint: disable=broad-exception-caught
+    except Exception as error:
+        log.error('retsic-prune %s', error)
+        return False
+    return True
 
+def run_check(rconf, args):
+    """restic check command"""
+    try:
+        status, _, stderr = run(rconf.exec_bin,
+                                'check',
+                                *rconf.base_args)
+        if not status:
+            log.error('restic-check %s', stderr)
+            return False
+    # pylint: disable=broad-exception-caught
+    except Exception as error:
+        log.error('restic-check %s', error)
+        return False
+    return True
 
-def run_watch():
-    logfile = open("restic.log","r")
-    loglines = follow(logfile, 'ERROR')
-    # TODO notification
-    for line in loglines:
-        print(line)
 
+# def follow(logfile, pattern):
+#     # catch up
+#     for line in logfile.readlines():
+#         if pattern in line:
+#             yield line
+#     logfile.seek(0, 2)
+#     while True:
+#         line = logfile.readline()
+#         if not line:
+#             time.sleep(0.1)
+#             continue
+#         if pattern in line:
+#             yield line
+#
+#
+# def run_watch(args):
+#     logfile = open("restic.log", "r")
+#     loglines = follow(logfile, 'ERROR')
+#     for line in loglines:
+#         print(line)
+#
 
 def main():
-    _check_auth()
-    cmd_arg = parser.parse_args()
-    cmd = 'run_{}'.format(cmd_arg.cmd)
+    """main"""
+    argz = prepare_args()
+    # ResticCmdConfig
+    restic_config = ResticCmdConfig(
+        resolve_restic(),
+        [
+            '--password-file', argz['password_file'],
+            '--repo', f'{argz["repo"]}/{argz["namespace"]}',
+        ],
+        [
+            '--exclude-file', argz['exclude_file'],
+        ]
+    )
     try:
-        run = globals()[cmd]
-    except KeyError:
-        log.error('command {} not found'.format(cmd))
+        success = argz['func'](restic_config, argz)
+    # pylint: disable=broad-exception-caught
+    except Exception as error:
+        log.error('error running %s', error)
+        sys.exit(1)
+    if not success:
         sys.exit(1)
-    run()
+    sys.exit(1)