diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/base/Dockerfile | 1 | ||||
| -rwxr-xr-x | src/base/bin/install_deb | 8 | ||||
| -rw-r--r-- | src/cgit/Dockerfile | 8 | ||||
| -rw-r--r-- | src/cgit/VERSION | 1 | ||||
| -rwxr-xr-x | src/cgit/bin/entrypoint.sh | 51 | ||||
| -rw-r--r-- | src/cgit/etc/uwsgi/cgit.ini | 10 | ||||
| -rw-r--r-- | src/gitsnapshot/Dockerfile | 4 | ||||
| -rw-r--r-- | src/gitsnapshot/README.md | 3 | ||||
| -rw-r--r-- | src/gitsnapshot/VERSION | 1 | ||||
| -rw-r--r-- | src/gitsnapshot/gitbackup.sh | 114 |
10 files changed, 201 insertions, 0 deletions
diff --git a/src/base/Dockerfile b/src/base/Dockerfile index e6f3db5..30366a7 100644 --- a/src/base/Dockerfile +++ b/src/base/Dockerfile @@ -5,4 +5,5 @@ RUN useradd -s /usr/sbin/nologin \ --create-home \ --home-dir /opt/app \ app +COPY bin/install_deb /usr/local/bin/install_deb WORKDIR /opt/app diff --git a/src/base/bin/install_deb b/src/base/bin/install_deb new file mode 100755 index 0000000..d367cfd --- /dev/null +++ b/src/base/bin/install_deb @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +DEBIAN_FRONTEND=noninteractive +APT_OPTS="--no-install-recommends --yes" +set -e +apt-get update --yes +exec apt-get install ${APT_OPTS} $@ +apt-get clean +rm -r /var/lib/apt/lists /var/cache/apt/archives diff --git a/src/cgit/Dockerfile b/src/cgit/Dockerfile new file mode 100644 index 0000000..45b5521 --- /dev/null +++ b/src/cgit/Dockerfile @@ -0,0 +1,8 @@ +FROM registry.gitlab.com/grumps/grumpy-containers/base:v0.0.2 +SHELL ["/bin/bash", "-c"] +RUN APT_OPTS="--no-install-recommends --install-suggests --yes" install_deb cgit uwsgi nginx-light \ + && mkdir -p /run/uwsgi \ + && chown www-data:www-data /run/uwsgi +COPY etc/uwsgi/cgit.ini /etc/uwsgi/cgit.ini +COPY bin/entrypoint.sh /usr/bin/entrypoint.sh +ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/src/cgit/VERSION b/src/cgit/VERSION new file mode 100644 index 0000000..a00f35a --- /dev/null +++ b/src/cgit/VERSION @@ -0,0 +1 @@ +v0.0.1-alpha diff --git a/src/cgit/bin/entrypoint.sh b/src/cgit/bin/entrypoint.sh new file mode 100755 index 0000000..bdd6edc --- /dev/null +++ b/src/cgit/bin/entrypoint.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +cat << EOF > /etc/nginx/sites-enabled/default +server { + listen $CGIT_PORT; + server_name $CGIT_HOSTNAME; + server_tokens off; + root /usr/share/cgit; + + # Serve static files with nginx + location ~* ^.+(cgit.(css|png)|favicon.ico|robots.txt) { + root /usr/share/cgit; + expires 30d; + } + location / { + gzip off; + include uwsgi_params; + uwsgi_modifier1 9; + uwsgi_pass unix:/run/uwsgi/cgit.sock; + } +} +EOF + +cat << EOF > /etc/cgitrc +# +# cgit config +# see cgitrc(5) for details + +root-title=git.ofmax.li +root-desc=grumpy software + +readme=:README.md +about-filter=/usr/lib/cgit/filters/about-formatting.sh +source-filter=/usr/lib/cgit/filters/syntax-highlighting.py + +enable-index-links=1 +enable-commit-graph=1 +enable-log-filecount=1 +enable-log-linecount=1 + +css=cgit.css +logo=cgit.png + + +project-list=/var/lib/git/projects.list +scan-path=/var/lib/git/repositories + +EOF + +uwsgi --daemonize true --ini /etc/uwsgi/cgit.ini > /dev/stdout 2> /dev/stderr < /dev/null & +nginx -c /etc/nginx/nginx.conf -g "daemon off;" diff --git a/src/cgit/etc/uwsgi/cgit.ini b/src/cgit/etc/uwsgi/cgit.ini new file mode 100644 index 0000000..85ac871 --- /dev/null +++ b/src/cgit/etc/uwsgi/cgit.ini @@ -0,0 +1,10 @@ +[uwsgi] +master = true +plugins = cgi +socket = /run/uwsgi/%n.sock +uid = www-data +gid = www-data +procname-master = uwsgi cgit +processes = 1 +threads = 2 +cgi = /usr/lib/cgit/cgit.cgi diff --git a/src/gitsnapshot/Dockerfile b/src/gitsnapshot/Dockerfile new file mode 100644 index 0000000..9df41d2 --- /dev/null +++ b/src/gitsnapshot/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3-slim-buster +RUN pip install awscli && apt-get update -y && apt-get install -y git && apt-get clean +COPY gitbackup.sh /opt/app/gitbackup.sh +RUN chmod +x /opt/app/gitbackup.sh diff --git a/src/gitsnapshot/README.md b/src/gitsnapshot/README.md new file mode 100644 index 0000000..603674f --- /dev/null +++ b/src/gitsnapshot/README.md @@ -0,0 +1,3 @@ +# gitbackup + +creates bundle, uploads to s3, creates a checkpoint for next iteration diff --git a/src/gitsnapshot/VERSION b/src/gitsnapshot/VERSION new file mode 100644 index 0000000..45c7a58 --- /dev/null +++ b/src/gitsnapshot/VERSION @@ -0,0 +1 @@ +v0.0.1 diff --git a/src/gitsnapshot/gitbackup.sh b/src/gitsnapshot/gitbackup.sh new file mode 100644 index 0000000..47dee97 --- /dev/null +++ b/src/gitsnapshot/gitbackup.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +AWS=/usr/local/bin/aws +basedir=$1 +bucket=$2 +repos_found=$basedir/*.git +num_repos_found=0 +completed=0 +backup_log="${basedir}/gitbackup-log" +mkdir -p "${backup_log}" +echo "starting for: $basedir" + +get_rev_list () { + : ' + generates a rev list since the last checkpoint. provides a blank + response if the checkpoint and the latest commit are one and the same + output: + "hash..hash2" + return + 0: successfully created a revlist + 1: no revlist created + ' + # last checkpoint in repo + local last_hash=$(git rev-list -n 1 --all) + # first commit in repo, or hash of last checkpoint + if ! $(git rev-list -n 1 CHECKPOINT &> /dev/null); + then + # first commit in repo + local checkpoint_hash=$(git rev-list --all | tail -n 1) + echo "${checkpoint_hash}" + return 0 + else + local checkpoint_hash=$(git rev-list -n 1 CHECKPOINT) + fi + + # if checkpoint and last hash are one and the same, offer nothing + if [ "${checkpoint_hash}" = "${last_hash}" ]; + then + # we provide nothing back + echo "" + return 1 + else + echo "${checkpoint_hash}..${last_hash}" + return 0 + fi +} + +create_bundle () { + : ' + Arguments: + arg1: rev_list in proper revlist format hash..hash2 + arg2: bundle name/path to create + creates a bundle for a given revlist + output: + none + returns: + 0: created bundle + 1: could not create bundle + ' + local rev_list=$1 + local bundle_path_name=$2 + if ! git bundle create "${bundle_path_name}" "${rev_list}" --all; + then + return 1 + else + return 0 + fi +} +for repo in $repos_found; +do + let num_repos_found++ || true + cd "${repo}" + repobase=$(basename $repo) + current_time=$(date +%s%N) + slug=${repobase%%.*} + tx_log="${backup_log}/${slug}.tx.log" + last_time=$(touch ${tx_log} && tail -n1 "${tx_log}" | cut -d " " -f 1) + last_time="${last_time:-0}" + bundle_name="${last_time}-${current_time}-${slug}.bundle" + bundle_tmp_path="/tmp/${bundle_name}" + s3_base="s3://${bucket}" + s3_uri="${s3_base}/${repobase}" + + rev_list=$(get_rev_list) + + if [ -z "${rev_list}" ]; + then + # we increase the counter, because not needing to update is ok + echo "${slug} has no new commits, no bundle generated" + let completed++ || true + continue + else + if ! $(create_bundle "${rev_list}" "${bundle_tmp_path}"); + then + echo "${bundle_name} failed to be created" >&2 + continue + fi + fi + if ! $AWS s3 cp "${bundle_tmp_path}" "${s3_uri}/${bundle_name}" + then + echo "$bundle_name failed to upload S3" >&2 + continue + else + # set checkpoint to the last hash in revlist + checkpoint="${rev_list:42}" + checkpoint=${checkpoint:-"${rev_list}"} + git tag -f CHECKPOINT "${checkpoint}" + # transaction log + echo "${current_time} ${rev_list}" >> "${tx_log}" + $AWS s3 cp "${tx_log}" "${s3_base}/logs/${slug}.tx.log" + fi + echo "${slug} backedup" + let completed++ || true +done +echo "${completed} of ${num_repos_found} repos backed up" |