Initial Commit

9 files changed, 492 insertions, 1 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..b710875
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+build/
+dist/
+*.egg-info/
+*.log
diff --git a/Pipfile b/Pipfile
new file mode 100644
index 0000000..4194aee
--- /dev/null
+++ b/Pipfile
@@ -0,0 +1,16 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[dev-packages]
+ipython = "*"
+ipdb = "*"
+"autopep8" = "*"
+
+[packages]
+sh = "*"
+restic = {path = "."}
+
+[requires]
+python_version = "3.6"
diff --git a/Pipfile.lock b/Pipfile.lock
new file mode 100644
index 0000000..20667e9
--- /dev/null
+++ b/Pipfile.lock
@@ -0,0 +1,155 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "ded3c2dd868d603f64fab2bdffb55751b970ba0a2ab481597f9cb52e4bdd0b1e"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.6"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "restic": {
+            "path": "."
+        },
+        "sh": {
+            "hashes": [
+                "sha256:ae3258c5249493cebe73cb4e18253a41ed69262484bad36fdb3efcb8ad8870bb",
+                "sha256:b52bf5833ed01c7b5c5fb73a7f71b3d98d48e9b9b8764236237bdc7ecae850fc"
+            ],
+            "index": "pypi",
+            "version": "==1.12.14"
+        }
+    },
+    "develop": {
+        "autopep8": {
+            "hashes": [
+                "sha256:1b8d42ebba751a91090d3adb5c06840b1151d71ed43e1c7a9ed6911bfe8ebe6c"
+            ],
+            "index": "pypi",
+            "version": "==1.4.2"
+        },
+        "backcall": {
+            "hashes": [
+                "sha256:38ecd85be2c1e78f77fd91700c76e14667dc21e2713b63876c0eb901196e01e4",
+                "sha256:bbbf4b1e5cd2bdb08f915895b51081c041bac22394fdfcfdfbe9f14b77c08bf2"
+            ],
+            "version": "==0.1.0"
+        },
+        "decorator": {
+            "hashes": [
+                "sha256:2c51dff8ef3c447388fe5e4453d24a2bf128d3a4c32af3fabef1f01c6851ab82",
+                "sha256:c39efa13fbdeb4506c476c9b3babf6a718da943dab7811c206005a4a956c080c"
+            ],
+            "version": "==4.3.0"
+        },
+        "ipdb": {
+            "hashes": [
+                "sha256:7081c65ed7bfe7737f83fa4213ca8afd9617b42ff6b3f1daf9a3419839a2a00a"
+            ],
+            "index": "pypi",
+            "version": "==0.11"
+        },
+        "ipython": {
+            "hashes": [
+                "sha256:a5781d6934a3341a1f9acb4ea5acdc7ea0a0855e689dbe755d070ca51e995435",
+                "sha256:b10a7ddd03657c761fc503495bc36471c8158e3fc948573fb9fe82a7029d8efd"
+            ],
+            "index": "pypi",
+            "version": "==7.1.1"
+        },
+        "ipython-genutils": {
+            "hashes": [
+                "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8",
+                "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"
+            ],
+            "version": "==0.2.0"
+        },
+        "jedi": {
+            "hashes": [
+                "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7",
+                "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148"
+            ],
+            "version": "==0.13.1"
+        },
+        "parso": {
+            "hashes": [
+                "sha256:35704a43a3c113cce4de228ddb39aab374b8004f4f2407d070b6a2ca784ce8a2",
+                "sha256:895c63e93b94ac1e1690f5fdd40b65f07c8171e3e53cbd7793b5b96c0e0a7f24"
+            ],
+            "version": "==0.3.1"
+        },
+        "pexpect": {
+            "hashes": [
+                "sha256:2a8e88259839571d1251d278476f3eec5db26deb73a70be5ed5dc5435e418aba",
+                "sha256:3fbd41d4caf27fa4a377bfd16fef87271099463e6fa73e92a52f92dfee5d425b"
+            ],
+            "markers": "sys_platform != 'win32'",
+            "version": "==4.6.0"
+        },
+        "pickleshare": {
+            "hashes": [
+                "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca",
+                "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"
+            ],
+            "version": "==0.7.5"
+        },
+        "prompt-toolkit": {
+            "hashes": [
+                "sha256:c1d6aff5252ab2ef391c2fe498ed8c088066f66bc64a8d5c095bbf795d9fec34",
+                "sha256:d4c47f79b635a0e70b84fdb97ebd9a274203706b1ee5ed44c10da62755cf3ec9",
+                "sha256:fd17048d8335c1e6d5ee403c3569953ba3eb8555d710bfc548faf0712666ea39"
+            ],
+            "version": "==2.0.7"
+        },
+        "ptyprocess": {
+            "hashes": [
+                "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0",
+                "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f"
+            ],
+            "version": "==0.6.0"
+        },
+        "pycodestyle": {
+            "hashes": [
+                "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83",
+                "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a"
+            ],
+            "version": "==2.4.0"
+        },
+        "pygments": {
+            "hashes": [
+                "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d",
+                "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc"
+            ],
+            "version": "==2.2.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
+                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+            ],
+            "version": "==1.11.0"
+        },
+        "traitlets": {
+            "hashes": [
+                "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835",
+                "sha256:c6cb5e6f57c5a9bdaa40fa71ce7b4af30298fbab9ece9815b5d995ab6217c7d9"
+            ],
+            "version": "==4.3.2"
+        },
+        "wcwidth": {
+            "hashes": [
+                "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
+                "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
+            ],
+            "version": "==0.1.7"
+        }
+    }
+}
diff --git a/README.md b/README.md
index f988468..bd06938 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,13 @@
-RESTIC WRAPPER
+# restic-py
+
+a stupid wrapper using `sh` module to run and monitor backups
+
+## Features
+
+### Large/Primary Directory seperation
+
+Seperates operations for key directories that are likely to have large numbers of files/disk spaces, this allows for better error handling, monitor and snapshot prunes.
+
+### Monitoring
+
+A continious monitoring service that allows real time monitoring of backups for errors
diff --git a/VERSION b/VERSION
new file mode 100644
index 0000000..351658d
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+0.01a
diff --git a/etc/exclude.conf b/etc/exclude.conf
new file mode 100644
index 0000000..e99ae21
--- /dev/null
+++ b/etc/exclude.conf
@@ -0,0 +1,137 @@
+.adobe
+.android
+.rvm
+.ansible
+.b
+.bash_history
+.bogofilter
+.bundler
+.bzr.log
+.cache
+.config
+.crashplan
+.dbus
+.dmrc
+.dropbox
+.dropbox-dist
+.drush
+en_US.UTF-8
+.filezilla
+.font.conf
+.font.conf~
+fontconfig
+.fontconfig
+.freerdp
+.galculator
+.gconf
+.gem
+.gimp-2.8
+.gksu.lock
+.gnome
+.gnome2
+.gnome2_private
+.gnupg
+.gphoto
+.gstreamer-0.10
+.gtk-bookmarks
+.gvfs
+hs_err_pid16032.log
+hs_err_pid25262.log
+hs_err_pid28743.log
+hs_err_pid30284.log
+hs_err_pid30643.log
+hs_err_pid32607.log
+hs_err_pid6334.log
+.ICEauthority
+.icedove
+.icons
+.ipython
+.java
+.kde
+.keybase
+.lesshst
+.libvirt
+.linkchecker
+.local
+.macromedia
+.mission-control
+.mono
+.mozilla
+.mysql_history
+.neocomplcache
+.netextender
+.netExtenderCerts
+netExtenderClient
+.netExtender.log
+.npm
+.obnam.conf
+.pgadmin_histoqueries
+pgadmin.log
+.pgpass
+.pida2
+.pip
+.pki
+playlist-results
+Podcasts
+prod.sql
+.profile
+.psql_history
+.pulse
+.pulse-cookie
+.purple
+.PyCharm20
+pycharm-3.0
+.PyCharm30
+pycharm-3.1.2
+.recently-used
+.remmina
+sane
+.sane
+.screamingfrogseospider
+.ScreamingFrogSEOSpider
+.selected_editor
+.shotwell
+.shutter
+.squirrel-sql
+.ssh
+.steam
+.steampath
+.steampid
+.subversion
+.swp
+.swt
+.thumbnails
+.uml
+.vagrant.d
+.vim
+.vim.20140331_1396321374
+.vimbackup
+.viminfo
+.vimrc
+.vimrc.before
+.vimrc.bundles
+.vimrc.local
+.vimswap
+.vimundo
+.vimviews
+.VirtualBox
+.virtualenvs
+.vnc
+X17-24209.iso
+.Xauthority
+.xchat2
+.xscreensaver
+.xsession
+.xsession-errors
+.xsession-errors.old
+.yolk
+.zcompdump
+.zprofile
+.zsh_history
+.zshrc
+.zshrc.bk
+.zsh-update
+Music/*
+.ievms/*
+Work/*
+chroots/*
diff --git a/restic.ini b/restic.ini
new file mode 100644
index 0000000..035447e
--- /dev/null
+++ b/restic.ini
@@ -0,0 +1,3 @@
+[restic]
+exclude=etc/exclude.conf
+repo_uri=sftp:vito.bing:/home/srv/grumps-repo
diff --git a/restic.py b/restic.py
new file mode 100644
index 0000000..8033103
--- /dev/null
+++ b/restic.py
@@ -0,0 +1,131 @@
+"""
+BSD 2-Clause License
+restic subprocess wrapper
+Copyright (c) 2018, Max Resnick <max@ofmax.li>
+All rights reserved.
+
+restic-py is simple wrapper to restic binary for backups and monitoring
+"""
+import os
+import tempfile
+import configparser
+import glob
+import argparse
+import logging
+import sys
+import time
+import io
+
+import sh
+
+# logging
+LOGLEVELS = {'DEBUG': logging.DEBUG,
+             'INFO': logging.INFO,
+             'WARNING': logging.WARNING,
+             'ERROR': logging.ERROR}
+# TODO should be a configurable location
+logging.basicConfig(filename='restic.log',
+                    level=LOGLEVELS[os.getenv('RESTIC_LOGLEVEL', 'WARNING')])
+log = logging.getLogger(__name__)
+# cmd line parser
+parser = argparse.ArgumentParser(description='wrapper to restic')
+parser.add_argument('cmd',
+                    type=str,
+                    choices=('backup', 'prune', 'check', 'watch'),
+                    help='run backup')
+
+# config
+cfg = configparser.ConfigParser()
+cfg.read('restic.ini')
+
+# baked commands
+HOME_DIR = os.path.expanduser('~')
+PASSWORD_FILE = '{}/.ssh/resticpass'.format(HOME_DIR)
+BACKUP_CMD = sh.restic.bake('backup',
+                            '--password-file', PASSWORD_FILE,
+                            '--exclude-file', cfg['restic']['exclude'],
+                            '--exclude-caches',
+                            '--repo', cfg['restic']['repo_uri'])
+PRUNE_CMD = sh.restic.bake('forget',
+                           '--password-file', PASSWORD_FILE,
+                           '--repo', cfg['restic']['repo_uri'],
+                           '--keep-daily', 7,
+                           '--keep-weekly', 5,
+                           '--keep-monthly', 12,
+                           '--keep-yearly', 75)
+CHECK_CMD = sh.restic.bake('check',
+                           '--password-file', PASSWORD_FILE,
+                           '--repo', cfg['restic']['repo_uri'])
+
+
+def _check_auth():
+    try:
+        os.environ['SSH_AUTH_SOCK']
+    except KeyError:
+        log.error('ssh agent not running')
+
+
+def run_backup():
+    directories = []
+    for chunk in glob.iglob('{}/*'.format(HOME_DIR)):
+        if os.path.isdir(chunk):
+            try:
+                out = BACKUP_CMD(chunk)
+                directories.append(chunk)
+                log.info('{} {}'.format('restic-run', chunk))
+            except sh.ErrorReturnCode_1 as e:
+                log.error('{} {} {}'.format('restic-run', chunk, e))
+    # backs up non chunked files
+    with tempfile.NamedTemporaryFile(delete=True) as tmp:
+        tmp.write('\n'.join(directories).encode())
+        out = BACKUP_CMD('--exclude-file', tmp.name, HOME_DIR)
+        log.info('{} {}'.format('restic-run', HOME_DIR)) 
+
+
+def run_prune():
+    try:
+        out = PRUNE_CMD()
+    except sh.ErrorReturnCode_1 as e:
+        log.error('{} {}'.format('restic-prune', e))
+
+
+def run_check():
+    try:
+        out = CHECK_CMD()
+    except sh.ErrorReturnCode_1 as e:
+        log.error('{} {}'.format('restic-check', e))
+
+
+def follow(logfile, pattern):
+    # catch up
+    for line in logfile.readlines():
+        if pattern in line:
+            yield line
+    logfile.seek(0, 2)
+    while True:
+        line = logfile.readline()
+        if not line:
+            time.sleep(0.1)
+            continue
+        if pattern in line:
+            yield line
+
+
+def run_watch():
+    logfile = open("restic.log","r")
+    loglines = follow(logfile, 'ERROR')
+    # TODO notification
+    for line in loglines:
+        print(line)
+
+
+def main():
+    _check_auth()
+    cmd_arg = parser.parse_args()
+    cmd = 'run_{}'.format(cmd_arg.cmd)
+    try:
+        run = globals()[cmd]
+    except KeyError:
+        log.error('command {} not found'.format(cmd))
+        sys.exit(1)
+    run()
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..226e1e8
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,32 @@
+"""setup"""
+
+from setuptools import setup
+
+
+with open('VERSION', 'r') as version_file:
+    version = version_file.read().strip()
+
+setup(name='restic',
+      version=version,
+      description='stupid wrapper to restic',
+      long_description=' '.join('''
+                                glues sh with restic
+                                '''.split()),
+      author='Max Resnick',
+      author_email='max@ofmax.li',
+      url='git@git.ofmax.li:restic_wrapper',
+      license='BSD',
+      py_modules=['restic'],
+      entry_points={
+        'console_scripts': ['rw=restic:main']
+      },
+      packages=[],
+      install_requires=['sh'],
+      classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Natural Language :: English',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python',
+        'Topic :: Utilities'])