diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 393290f..0dc5e09 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,3 +1,4 @@ +--- # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: @@ -5,14 +6,9 @@ version: 2 updates: - - package-ecosystem: "docker" # See documentation for possible values - directory: "/" # Location of package manifests + - package-ecosystem: "github-actions" + directory: "/" assignees: - Elliot Matson schedule: interval: "daily" - - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "daily" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..f05b797 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,64 @@ +--- +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# GitHub recommends pinning actions to a commit SHA. +# To get a newer version, you will need to update the SHA. +# You can also reference a tag or branch, +# but the action may change without warning. + +name: Create and publish a Docker image + +on: + push: + branches: ['release', 'dev'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: elliotmatson/pgadmin-config-creator + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: ./pgadmin-config-creator + platforms: linux/amd64,linux/arm64,linux/arm/v7 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0bc142a..e6dd1de 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,9 +10,7 @@ jobs: run-lint: name: Lint runs-on: ubuntu-latest - steps: - - name: Check out the codebase. uses: actions/checkout@v3 with: diff --git a/.github/workflows/stack-healthcheck.yml b/.github/workflows/stack-healthcheck.yml new file mode 100644 index 0000000..28091ca --- /dev/null +++ b/.github/workflows/stack-healthcheck.yml @@ -0,0 +1,21 @@ +--- +name: Stack Healthcheck +on: + push: + pull_request: + workflow_dispatch: + +jobs: + test: + name: Test Stack + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Deploy the stack, timeout if not healthy after 2m + run: timeout 120 docker compose up --quiet-pull --wait + - name: Check creating database + run: docker compose logs | grep "CREATE DATABASE" + - name: Check database creation + run: docker compose logs | grep "Success. You can now start the database server" + - name: Check postgres init + run: docker compose logs | grep "PostgreSQL init process complete; ready for start up." \ No newline at end of file diff --git a/README.md b/README.md index 6491010..c0ce524 100644 --- a/README.md +++ b/README.md @@ -7,16 +7,16 @@ Simple Resolve project server with automatic backups - [Introduction](#introduction) - [Features](#features) - [Configuration](#configuration) - - [PostgreSQL Server](#postgresql-server) - - [Backup Server](#backup-server) + - [Database](#database) + - [Backups](#backups) + - [PGAdmin](#pgadmin) + - [Volume Locations](#volume-locations) - [Installation](#installation) - [QNAP Installation](#qnap-installation) - [Synology](#synology) - [Linux](#linux) - [Different PostgreSQL versions](#different-postgresql-versions) - - [Setting up a PostgreSQL 11 Project Server](#setting-up-a-postgresql-11-project-server) - - [Using PostgreSQL 11 on Windows](#using-postgresql-11-on-windows) - - [Using PostgreSQL 11 on Mac](#using-postgresql-11-on-mac) + - [Setting up a PostgreSQL 9.5 or 11 Project Server](#setting-up-a-postgresql-95-or-11-project-server) - [Thanks](#thanks) ## Introduction @@ -27,83 +27,72 @@ There are a lot of ways to host a Resolve project server, but each of them has t - **Lightweight** - Docker based, so doesn't require a full macOS or Windows machine or VM. - **Platform Independent** - can be installed on Windows, Mac, Linux, QNAP, Synology, RPi, really anything that can run Docker. - **Compatible with Resolve's existing backup/restore functions** - All backup files use the standard Resolve *.backup file syntax, and can be restored from the Resolve UI +- **Built-in PGAdmin Server** - PGAdmin is a tool for administering a PostgreSQL Server, and is helpful for diagnosing problems and migrating/updating entire servers ## Configuration -There are a few things we'll need to edit in the docker-compose.yml file to configure our installation: -### PostgreSQL Server +There are a few things we'll need to edit at the top of the docker-compose.yml file to configure our installation: +```yaml +--- +version: '3.8' +x-common: + database: &db-environment + POSTGRES_DB: database + POSTGRES_USER: postgres + POSTGRES_PASSWORD: DaVinci + TZ: America/Chicago + POSTGRES_LOCATION: &db-location "???:/var/lib/postgresql/data" + backup: &backup-environment + SCHEDULE: "@daily" + BACKUP_KEEP_DAYS: 7 + BACKUP_KEEP_WEEKS: 4 + BACKUP_KEEP_MONTHS: 6 + BACKUP_LOCATION: &bk-location "???:/backups" + admin: &admin-environment + PGADMIN_DEFAULT_EMAIL: admin@admin.com + PGADMIN_DEFAULT_PASSWORD: root + PGADMIN_PORT: &pgadmin-port "3001:80" +... +``` + +### Database To configure the server itself, we'll want to configure the environment variables below: -```yaml -... -services: - postgres: - ... - environment: - - POSTGRES_DB=database - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=DaVinci - - TZ=America/Chicago - ... -... -``` | Environment Variable |Meaning| |---|---| -| POSTGRES_DB | This is the name of your database. Name it whatever you like. | -| POSTGRES_USER | This is the username you will use to connect to your database. The Resolve default is "postgres" | -| POSTGRES_PASSWORD | This is the password you will use to connect to your database. The Resolve default is "DaVinci" | -| TZ | This is your timezone, here is [a list](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)| +| POSTGRES_DB | Nme of your database. Name it whatever you like. | +| POSTGRES_USER | Username you will use to connect to your database. The Resolve default is "postgres" | +| POSTGRES_PASSWORD | Password you will use to connect to your database. The Resolve default is "DaVinci" | +| TZ | Your timezone, here is [a list](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)| +| POSTGRES_LOCATION | Location your database will be stored. See [Volume Locations](#volume-locations) | -### Backup Server +### Backups To configure the backups, we'll want to configure the variables below: -```yaml -... -services: - ... - pgbackups: - ... - volumes: - - "(Whatever location you want backups stored):/backups" - ... - environment: - - POSTGRES_HOST=postgres - - POSTGRES_DB=database - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=DaVinci - - POSTGRES_EXTRA_OPTS=--blobs --format=custom --quote-all-identifiers - - BACKUP_SUFFIX=.backup - - SCHEDULE=@daily - - BACKUP_KEEP_DAYS=7 - - BACKUP_KEEP_WEEKS=4 - - BACKUP_KEEP_MONTHS=6 - - HEALTHCHECK_PORT=8080 - - TZ=America/Chicago - ... -... -``` -First, we will want to decide on a backup location and edit the ```volumes:``` section. You will need the full path to the folder you want backups stored in. On a QNAP NAS for example, if I wanted to use a folder called "Backups" inside a shared folder named "Videos", the path would be ```/shares/Videos/Backups/```, and my ```volumes:``` section would look like this: -```yaml -volumes: - - "/shares/Videos/Backups:/backups" -``` - On Ubuntu, if I wanted to use a folder named "Backups" in the home directory of the user named "johndoe", the path would be ```/home/johndoe/Backups/```, and my ```volumes:``` section would look like this: -```yaml -volumes: - - "/home/johndoe/Backups:/backups" -``` - - - - -There are also some variables in the environment section. Many of these don't need to be edited, but here are the ones you might want to change: | Environment Variable |Meaning| |---|---| -| POSTGRES_DB | This is the name of the database from the previous step. Can also be a comma/space separated list of database names if you create more in the future | -| POSTGRES_USER | This is the database username from the previous step | -| POSTGRES_PASSWORD | This is the database password from the previous step | -| SCHEDULE | This is a [cron string](https://www.freeformatter.com/cron-expression-generator-quartz.html) for how often backups are created. can be "@daily", "@every 1h", etc | -| BACKUP_KEEP_DAYS | Number of daily backups to keep before removal. -| BACKUP_KEEP_WEEKS | Number of weekly backups to keep before removal. -| BACKUP_KEEP_MONTHS | Number of daily backups to keep before removal. -| TZ | This is your timezone, here is [a list](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)| +| SCHEDULE | This is a [cron string](https://www.freeformatter.com/cron-expression-generator-quartz.html) for how often backups are created. can be "@daily", "@every 1h", etc | +| BACKUP_KEEP_DAYS | Number of daily backups to keep before removal. | +| BACKUP_KEEP_WEEKS | Number of weekly backups to keep before removal. | +| BACKUP_KEEP_MONTHS | Number of daily backups to keep before removal. | +| BACKUP_LOCATION | Location your backups will be stored. See [Volume Locations](#volume-locations) | + +### PGAdmin +To configure PGAdmin, we'll want to configure the variables below: +| Environment Variable |Meaning| +|---|---| +| PGADMIN_DEFAULT_EMAIL | Email used for PGAdmin login. Default is "admin@admin.com" | +| PGADMIN_DEFAULT_PASSWORD | Password used for PGAdmin login. Default is "root" | +| PGADMIN_PORT | String configuring port to expose PGAdmin on. Syntax is "YOUR_PORT:80" | + +### Volume Locations +The location of your database and backups depend on what platform you are installing on. You will need the full path to the folder you want them stored in. On a QNAP NAS for example, if I wanted to use a folder called "Backups" inside a shared folder named "Videos" for my backups location, the path would be ```/shares/Videos/Backups/```, and my ```BACKUP_LOCATION``` value would look like this: +```yaml +BACKUP_LOCATION: &bk-location "/shares/Videos/Backups/:/backups" +``` + On Ubuntu, if I wanted to use a folder named "database" in the home directory of the user named "johndoe" for my database location, the path would be ```/home/johndoe/database/```, and my ```POSTGRES_LOCATION``` value would look like this: +```yaml +POSTGRES_LOCATION: &db-location "johndoe/database/:/var/lib/postgresql/data" +``` + +I recommend putting your database on an SSD, your access speed will be noticeably slower on a spinning drive. Once you have configured these settings, save your modified docker-compose.yml file and move on to installation! diff --git a/docker-compose.yml b/docker-compose.yml index 154b9ce..3064056 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,39 +1,85 @@ --- -version: '3' +version: '3.8' +x-common: + database: &db-environment + POSTGRES_DB: database + POSTGRES_USER: postgres + POSTGRES_PASSWORD: DaVinci + TZ: America/Chicago + POSTGRES_LOCATION: &db-location "~/db:/var/lib/postgresql/data" + backup: &backup-environment + SCHEDULE: "@daily" + BACKUP_KEEP_DAYS: 7 + BACKUP_KEEP_WEEKS: 4 + BACKUP_KEEP_MONTHS: 6 + BACKUP_LOCATION: &bk-location "~/backups:/backups" + admin: &admin-environment + PGADMIN_DEFAULT_EMAIL: admin@admin.com + PGADMIN_DEFAULT_PASSWORD: root + PGADMIN_PORT: &pgadmin-port "3001:80" +# +# ------------------------------------------------------------------------------------------ +# DANGER ZONE BELOW +# +# The remainder of this file likely does not need to be changed. +# Please only make modifications +# below if you understand what you are doing. +# services: postgres: + container_name: resolve_pgsql image: postgres:13 restart: always ports: - "5432:5432" environment: - - POSTGRES_DB=database - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=DaVinci - - TZ=America/Chicago + <<: [*db-environment] volumes: - - "resolve-db:/var/lib/postgresql/data" + - *db-location + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 10s + timeout: 5s + retries: 5 pgbackups: + container_name: resolve_pgbackup image: prodrigestivill/postgres-backup-local:13 restart: always volumes: - - "(location you want backups stored):/backups" - links: - - postgres + - *bk-location depends_on: - postgres environment: - - POSTGRES_HOST=postgres - - POSTGRES_DB=database - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=DaVinci - - POSTGRES_EXTRA_OPTS=--blobs --format=custom --quote-all-identifiers - - BACKUP_SUFFIX=.backup - - SCHEDULE=@daily - - BACKUP_KEEP_DAYS=7 - - BACKUP_KEEP_WEEKS=4 - - BACKUP_KEEP_MONTHS=6 - - HEALTHCHECK_PORT=8080 - - TZ=America/Chicago + <<: [*db-environment, *backup-environment] + POSTGRES_HOST: postgres + POSTGRES_EXTRA_OPTS: --blobs --format=custom --quote-all-identifiers + BACKUP_SUFFIX: .backup + HEALTHCHECK_PORT: 8080 + healthcheck: + interval: 30s + pgadmin: + container_name: resolve_pgadmin + image: dpage/pgadmin4 + restart: always + environment: + <<: [*admin-environment] + ports: + - *pgadmin-port + volumes: + - pgadmin-config:/pgadmin4 + depends_on: + - postgres + - pgadmin-config + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost/ || exit 1"] + pgadmin-config: + container_name: resolve_pgadmin-config-creator + image: ghcr.io/elliotmatson/pgadmin-config-creator:latest + restart: on-failure + environment: + <<: [*db-environment] + volumes: + - pgadmin-config:/config + volumes: - resolve-db: + pgadmin-config: diff --git a/pgadmin-config-creator/Dockerfile b/pgadmin-config-creator/Dockerfile new file mode 100644 index 0000000..22739a4 --- /dev/null +++ b/pgadmin-config-creator/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.8-alpine +RUN mkdir /app +RUN mkdir /config +COPY . /app +WORKDIR /app +CMD ["python", "app.py"] +HEALTHCHECK CMD test -f /config/servers.json || exit 1 \ No newline at end of file diff --git a/pgadmin-config-creator/app.py b/pgadmin-config-creator/app.py new file mode 100644 index 0000000..9cd2048 --- /dev/null +++ b/pgadmin-config-creator/app.py @@ -0,0 +1,18 @@ +import json +import os +import time + +with open("/config/servers.json", "w") as f: + print("created " + f.name) + template_string = ( + '{"Servers": {"1": {"Name": "DavinciResolve", ' + '"Group": "Servers", "Port": 5432, "Username": ' + '"postgres", "Host": "postgres", "SSLMode": "prefer", "MaintenanceDB": "postgres"}}}' + ) + data = json.loads(template_string) + data["Servers"]["1"]["Username"] = os.getenv("POSTGRES_USER") + print("printing json to file...") + print(data) + json.dump(data, f, indent=2) +# sleep until GHA Healthchecks are complete +time.sleep(125)