diff --git a/MigrationDockerfile b/MigrationDockerfile index fa8f671..1b71a21 100644 --- a/MigrationDockerfile +++ b/MigrationDockerfile @@ -11,6 +11,8 @@ COPY backend-legacy/tsconfig.json /app/legacy/ COPY backend-legacy/package.json /app/legacy/ COPY backend-legacy/package-lock.json /app/legacy/ WORKDIR /app/legacy +RUN apt update +RUN apt install postgresql -y RUN npm install --save-dev RUN npm run build RUN rm out/**/*.ts out/**/*.map diff --git a/backend/api/backup.go b/backend/api/backup.go index f53cbe7..37e9ffa 100644 --- a/backend/api/backup.go +++ b/backend/api/backup.go @@ -2,19 +2,15 @@ package main import ( "compress/gzip" - "context" "encoding/json" - "errors" "fmt" "io" "os" "strings" - "time" "git.terah.dev/imterah/hermes/api/dbcore" "github.com/charmbracelet/log" "github.com/go-playground/validator/v10" - "github.com/jackc/pgx/v5" "github.com/urfave/cli/v2" "gorm.io/gorm" ) @@ -93,10 +89,15 @@ func backupRestoreEntrypoint(cCtx *cli.Context) error { } reader, err := gzip.NewReader(backupFile) + + if err != nil { + return fmt.Errorf("failed to initialize Gzip (compression) reader: %s", err.Error()) + } + backupDataBytes, err := io.ReadAll(reader) if err != nil { - log.Fatal(err) + return fmt.Errorf("failed to read backup contents: %s", err.Error()) } log.Info("Decompressed backup. Cleaning up...") @@ -127,73 +128,7 @@ func backupRestoreEntrypoint(cCtx *cli.Context) error { return fmt.Errorf("failed to validate backup: %s", err.Error()) } - log.Warn("!! WARNING !!") - log.Warn("This will attempt to permanently wipe the old database. The backup will not be deleted, however, caution is still advised.") - log.Warn("Continuing in 5 seconds...") - - time.Sleep(5 * time.Second) - - log.Info("Wiping database...") - - databaseBackend := os.Getenv("HERMES_DATABASE_BACKEND") - - switch databaseBackend { - case "sqlite": - filePath := os.Getenv("HERMES_SQLITE_FILEPATH") - - if filePath == "" { - return fmt.Errorf("sqlite database file not specified (missing HERMES_SQLITE_FILEPATH)") - } - - err := os.Remove(filePath) - - if err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("failed to delete sqlite database: %s", err.Error()) - } - case "postgresql": - // FIXME(imterah): Maybe make this not required? - postgresDB := os.Getenv("HERMES_MIGRATE_POSTGRES_DATABASE") - - if postgresDB == "" { - return fmt.Errorf("postgres migration DB is not specified (we don't parse the DSN to save space) (missing HERMES_MIGRATE_POSTGRES_DATABASE)") - } - - postgresDSN := os.Getenv("HERMES_POSTGRES_DSN") - - if postgresDSN == "" { - return fmt.Errorf("postgres DSN not specified (missing HERMES_POSTGRES_DSN)") - } - - log.Info("Connecting to database...") - - db, err := pgx.Connect(context.Background(), postgresDSN) - - if err != nil { - return fmt.Errorf("failed to connect to database: %s", err.Error()) - } - - log.Info("Dropping database...") - - _, err = db.Query(context.Background(), fmt.Sprintf("DROP DATABASE %s", pgx.Identifier{postgresDB}.Sanitize())) - - if err != nil { - return fmt.Errorf("failed to drop database: %s", err.Error()) - } - - log.Info("Closing database connection...") - - err = db.Close(context.Background()) - - if err != nil { - return fmt.Errorf("failed to close database connection: %s", err.Error()) - } - case "": - return fmt.Errorf("no database backend specified in environment variables (missing HERMES_DATABASE_BACKEND)") - default: - return fmt.Errorf("unknown database backend specified: %s", os.Getenv(databaseBackend)) - } - - log.Info("Reinitializing database and opening it...") + log.Info("Initializing database and opening it...") err = dbcore.InitializeDatabase(&gorm.Config{}) @@ -254,7 +189,7 @@ func backupRestoreEntrypoint(cCtx *cli.Context) error { var bestEffortOwnerUID uint for _, user := range backupData.Users { - log.Debugf("Migrating user with email '%s' and ID of '%d'", user.Email, user.ID) + log.Infof("Migrating user with email '%s' and ID of '%d'", user.Email, user.ID) tokens := make([]dbcore.Token, 0) permissions := make([]dbcore.Permission, 0) @@ -307,7 +242,7 @@ func backupRestoreEntrypoint(cCtx *cli.Context) error { } for _, backend := range backupData.Backends { - log.Debugf("Migrating backend ID '%d' with name '%s'", backend.ID, backend.Name) + log.Infof("Migrating backend ID '%d' with name '%s'", backend.ID, backend.Name) backendDatabase := &dbcore.Backend{ UserID: bestEffortOwnerUID, @@ -321,14 +256,14 @@ func backupRestoreEntrypoint(cCtx *cli.Context) error { log.Errorf("Failed to create backend: %s", err.Error()) } - log.Debugf("Migrating proxies for backend ID '%d'", backend.ID) + log.Infof("Migrating proxies for backend ID '%d'", backend.ID) for _, proxy := range backupData.Proxies { if proxy.BackendID != backend.ID { continue } - log.Debugf("Migrating proxy ID '%d' with name '%s'", proxy.ID, proxy.Name) + log.Infof("Migrating proxy ID '%d' with name '%s'", proxy.ID, proxy.Name) proxyDatabase := &dbcore.Proxy{ BackendID: backendDatabase.ID, diff --git a/docs/nextnet_to_hermes_migration.md b/docs/nextnet_to_hermes_migration.md index 716fd06..7c3d2b3 100644 --- a/docs/nextnet_to_hermes_migration.md +++ b/docs/nextnet_to_hermes_migration.md @@ -17,10 +17,11 @@ Below are new environment variables that may need to be set up: ## Migration steps 1. Remove all old environment variables. 2. Add these variables: - - `HERMES_MIGRATE_POSTGRES_DATABASE` -> `$POSTGRES_DB` + - `HERMES_MIGRATE_POSTGRES_DATABASE` -> `${POSTGRES_DB}` - `HERMES_DATABASE_BACKEND` -> `postgresql` - - `HERMES_POSTGRES_DSN` -> `postgres://$POSTGRES_USERNAME:$POSTGRES_PASSWORD@nextnet-postgres:5432/$POSTGRES_DB` - - `DATABASE_URL` -> `postgresql://$POSTGRES_USERNAME:$POSTGRES_PASSWORD@nextnet-postgres:5432/$POSTGRES_DB?schema=nextnet` + - `HERMES_POSTGRES_DSN` -> `postgres://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@nextnet-postgres:5432/${POSTGRES_DB}` + - `DATABASE_URL` -> `postgresql://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@nextnet-postgres:5432/${POSTGRES_DB}?schema=nextnet` + - `HERMES_JWT_SECRET` -> Random data (recommended to use `head -c 500 /dev/random | sha512sum | cut -d " " -f 1` to seed the data) 3. Switch the API docker image from `ghcr.io/imterah/nextnet:latest` to `ghcr.io/imterah/hermes-backend-migration:latest` 4. Change the exposed ports from `3000:3000` to `3000:8000`. 5. Start the Docker compose stack. diff --git a/migration-entrypoint.sh b/migration-entrypoint.sh index d07b20f..1904e2b 100755 --- a/migration-entrypoint.sh +++ b/migration-entrypoint.sh @@ -1,10 +1,17 @@ #!/usr/bin/env bash -echo "Welcome to the Hermes migration wizard." +echo "Welcome to the Hermes migration assistant." if [ ! -f "/tmp/db.json.gz" ]; then echo "Exporting database contents..." cd /app/legacy node out/tools/exportDBContents.js /tmp/db.json.gz + BACKUP_EXIT_CODE=$? + + if [ $BACKUP_EXIT_CODE -ne 0 ]; then + echo "Failed to export database contents!" + exit 1 + fi + echo "!! IMPORTANT !!" echo "Database backup contents below:" echo "==== BEGIN BACKUP ====" @@ -13,11 +20,21 @@ if [ ! -f "/tmp/db.json.gz" ]; then echo "When copying, do NOT copy the BEGIN and END sections." fi +echo "Wiping old database..." +cat >> /tmp/wipe.sql << EOF +CREATE DATABASE temp; +\c temp +DROP DATABASE $HERMES_MIGRATE_POSTGRES_DATABASE; +CREATE DATABASE $HERMES_MIGRATE_POSTGRES_DATABASE; +\c nextnet +DROP DATABASE temp; +EOF +psql "$HERMES_POSTGRES_DATABASE" < /tmp/wipe.sql +rm -rf /tmp/wipe.sql echo "Restoring backup..." cd /app/modern ./hermes -b ./backends.json import --bp /tmp/db.json.gz -rm -rf /tmp/db.json.gz echo "Restored backup. If this restore fails after the database has wiped, get a shell into the container," echo "copy the backup contents into the container (base64 decoded) at '/tmp/db.json.gz',"