Move database schema updating to gochan-migration

build.py

@@ -39,6 +39,7 @@ release_files = (
 )
 
 GOCHAN_VERSION = "3.5.1"
+DATABASE_VERSION = "2" # stored in DBNAME.DBPREFIXdatabase_version
 
 PATH_NOTHING = -1
 PATH_UNKNOWN = 0

cmd/gochan-migration/internal/common/sqlutil.go

@@ -24,7 +24,7 @@ func RunSQLFile(path string, db *gcsql.GCDB) error {
 	sqlArr := strings.Split(sqlStr, ";")
 
 	for _, statement := range sqlArr {
-		statement = strings.Trim(statement, " \n\r\t")
+		statement = strings.TrimSpace(statement)
 		if len(statement) > 0 {
 			if _, err = db.ExecSQL(statement); err != nil {
 				return err

cmd/gochan-migration/internal/gcupdate/gcupdate.go

@@ -0,0 +1,184 @@
+package gcupdate
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
+	"github.com/gochan-org/gochan/pkg/config"
+	"github.com/gochan-org/gochan/pkg/gcsql"
+	"github.com/gochan-org/gochan/pkg/gcutil"
+)
+
+const (
+	// if the database version is less than this, it is assumed to be out of date, and the schema needs to be adjusted
+	latestDatabaseVersion = 2
+)
+
+type GCDatabaseUpdater struct {
+	options *common.MigrationOptions
+	db      *gcsql.GCDB
+}
+
+func (dbu *GCDatabaseUpdater) Init(options *common.MigrationOptions) error {
+	dbu.options = options
+	criticalCfg := config.GetSystemCriticalConfig()
+	var err error
+	dbu.db, err = gcsql.Open(
+		criticalCfg.DBhost, criticalCfg.DBtype, criticalCfg.DBname, criticalCfg.DBusername, criticalCfg.DBpassword,
+		criticalCfg.DBprefix,
+	)
+	return err
+}
+
+func (dbu *GCDatabaseUpdater) IsMigrated() (bool, error) {
+	var currentDatabaseVersion int
+	err := dbu.db.QueryRowSQL(`SELECT version FROM DBPREFIXdatabase_version WHERE component = 'gochan'`, nil,
+		[]any{&currentDatabaseVersion})
+	if err != nil {
+		return false, err
+	}
+	if currentDatabaseVersion == latestDatabaseVersion {
+		return true, nil
+	}
+	if currentDatabaseVersion > latestDatabaseVersion {
+		return false, fmt.Errorf("database layout is ahead of current version (%d), target version: %d",
+			currentDatabaseVersion, latestDatabaseVersion)
+	}
+	return false, nil
+}
+
+func (dbu *GCDatabaseUpdater) MigrateDB() (bool, error) {
+	migrated, err := dbu.IsMigrated()
+	if migrated || err != nil {
+		return migrated, err
+	}
+
+	var query string
+	criticalConfig := config.GetSystemCriticalConfig()
+	ctx := context.Background()
+	tx, err := dbu.db.BeginTx(ctx, &sql.TxOptions{
+		Isolation: 0,
+		ReadOnly:  false,
+	})
+	if err != nil {
+		return false, err
+	}
+	defer tx.Rollback()
+
+	switch criticalConfig.DBtype {
+	case "mysql":
+		query = `SELECT COUNT(*) FROM information_schema.TABLE_CONSTRAINTS
+		WHERE CONSTRAINT_NAME = 'wordfilters_board_id_fk'
+		AND TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'DBPREFIXwordfilters'`
+		var numConstraints int
+
+		if err = dbu.db.QueryRowTxSQL(tx, query, nil, []any{&numConstraints}); err != nil {
+			return false, err
+		}
+		if numConstraints > 0 {
+			query = `ALTER TABLE DBPREFIXwordfilters DROP FOREIGN KEY wordfilters_board_id_fk`
+		} else {
+			query = ""
+		}
+		query = `SELECT COUNT(*) FROM information_schema.COLUMNS
+		WHERE TABLE_SCHEMA = DATABASE()
+		AND TABLE_NAME = 'DBPREFIXwordfilters'
+		AND COLUMN_NAME = 'board_dirs'`
+		var numColumns int
+		if err = dbu.db.QueryRowTxSQL(tx, query, nil, []any{&numColumns}); err != nil {
+			return false, err
+		}
+		if numColumns == 0 {
+			query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
+			if _, err = gcsql.ExecTxSQL(tx, query); err != nil {
+				return false, err
+			}
+		}
+
+		// Yay, collation! Everybody loves MySQL's default collation!
+		query = `ALTER DATABASE ` + criticalConfig.DBname + ` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci`
+		if _, err = tx.Exec(query); err != nil {
+			return false, err
+		}
+
+		query = `SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?`
+		rows, err := dbu.db.QuerySQL(query, criticalConfig.DBname)
+		if err != nil {
+			return false, err
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var tableName string
+			err = rows.Scan(&tableName)
+			if err != nil {
+				return false, err
+			}
+			query = `ALTER TABLE ` + tableName + ` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`
+			if _, err = tx.Exec(query); err != nil {
+				return false, err
+			}
+		}
+		err = nil
+	case "postgres":
+		_, err = gcsql.ExecSQL(`ALTER TABLE DBPREFIXwordfilters DROP CONSTRAINT IF EXISTS board_id_fk`)
+		if err != nil {
+			return false, err
+		}
+		query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN IF NOT EXISTS board_dirs varchar(255) DEFAULT '*'`
+		if _, err = dbu.db.ExecTxSQL(tx, query); err != nil {
+			return false, err
+		}
+	case "sqlite3":
+		_, err = gcsql.ExecSQL(`PRAGMA foreign_keys = ON`)
+		if err != nil {
+			return false, err
+		}
+		query = `SELECT COUNT(*) FROM PRAGMA_TABLE_INFO('DBPREFIXwordfilters') WHERE name = 'board_dirs'`
+		var numColumns int
+		if err = dbu.db.QueryRowSQL(query, nil, []any{&numColumns}); err != nil {
+			return false, err
+		}
+		if numColumns == 0 {
+			query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
+			if _, err = dbu.db.ExecTxSQL(tx, query); err != nil {
+				return false, err
+			}
+		}
+	}
+
+	query = `UPDATE DBPREFIXdatabase_version SET version = ? WHERE component = 'gochan'`
+	_, err = dbu.db.ExecTxSQL(tx, query, latestDatabaseVersion)
+	if err != nil {
+		return false, err
+	}
+	return false, tx.Commit()
+}
+
+func (dbu *GCDatabaseUpdater) MigrateBoards() error {
+	return gcutil.ErrNotImplemented
+}
+
+func (dbu *GCDatabaseUpdater) MigratePosts() error {
+	return gcutil.ErrNotImplemented
+}
+
+func (dbu *GCDatabaseUpdater) MigrateStaff(password string) error {
+	return gcutil.ErrNotImplemented
+}
+
+func (dbu *GCDatabaseUpdater) MigrateBans() error {
+	return gcutil.ErrNotImplemented
+}
+
+func (dbu *GCDatabaseUpdater) MigrateAnnouncements() error {
+	return gcutil.ErrNotImplemented
+}
+
+func (dbu *GCDatabaseUpdater) Close() error {
+	if dbu.db != nil {
+		return dbu.db.Close()
+	}
+	return nil
+}

cmd/gochan-migration/main.go

@@ -3,8 +3,10 @@ package main
 import (
 	"flag"
 	"log"
+	"os"
 
 	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/gcupdate"
 	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/pre2021"
 	"github.com/gochan-org/gochan/pkg/config"
 
@@ -13,7 +15,7 @@ import (
 
 const (
 	banner = `Welcome to the gochan database migration tool for gochan %s!
-This migration tool is currently very unstable, and will likely go through
+This migration tool is currently unstable, and will likely go through
 several changes before it can be considered "stable", so make sure you check
 the README and/or the -h command line flag before you use it.
 
@@ -29,13 +31,16 @@ for the threads and board pages`
 
 var (
 	versionStr   string
+	dbVersionStr string
 )
 
 func main() {
 	var options common.MigrationOptions
 	var dirAction string
+	var updateDB bool
 
 	log.SetFlags(0)
+	flag.BoolVar(&updateDB, "updatedb", false, "If this is set, gochan-migrate will check, and if needed, update gochan's database schema")
 	flag.StringVar(&options.ChanType, "oldchan", "", "The imageboard we are migrating from (currently only pre2021 is supported, but more are coming")
 	flag.StringVar(&options.OldChanConfig, "oldconfig", "", "The path to the old chan's configuration file")
 	// flag.StringVar(&dirAction, "diraction", "", "Action taken on each board directory after it has been migrated. "+allowedDirActions)
@@ -43,10 +48,12 @@ func main() {
 
 	config.InitConfig(versionStr)
 
-	if options.ChanType == "" || options.OldChanConfig == "" {
+	if !updateDB && (options.ChanType == "" || options.OldChanConfig == "") {
 		flag.PrintDefaults()
 		log.Fatal("Missing required oldchan value")
 		return
+	} else if updateDB {
+		options.ChanType = "gcupdate"
 	}
 	switch dirAction {
 	case "":
@@ -64,6 +71,8 @@ func main() {
 	log.Printf(banner, versionStr)
 	var migrator common.DBMigrator
 	switch options.ChanType {
+	case "gcupdate":
+		migrator = &gcupdate.GCDatabaseUpdater{}
 	case "pre2021":
 		migrator = &pre2021.Pre2021Migrator{}
 	case "kusabax":
@@ -77,9 +86,10 @@ func main() {
 		return
 	}
 	config.InitConfig(versionStr)
+	var err error
+	if !updateDB {
 		systemCritical := config.GetSystemCriticalConfig()
-
-	err := gcsql.ConnectToDB(
+		err = gcsql.ConnectToDB(
 			systemCritical.DBhost, systemCritical.DBtype, systemCritical.DBname,
 			systemCritical.DBusername, systemCritical.DBpassword, systemCritical.DBprefix)
 		if err != nil {
@@ -89,6 +99,7 @@ func main() {
 			log.Fatalf("Failed to initialize the database: %s", err.Error())
 		}
 		defer gcsql.Close()
+	}
 
 	if err = migrator.Init(&options); err != nil {
 		log.Fatalf("Unable to initialize %s migrator: %s\n",
@@ -102,7 +113,8 @@ func main() {
 		log.Fatalln("Error migrating database:", err.Error())
 	}
 	if migrated {
-		log.Fatalf("Database is already migrated")
+		log.Println("Database is already migrated")
+		os.Exit(0)
 	}
 	log.Println(migrateCompleteTxt)
 }

pkg/gcsql/connect.go

@@ -4,8 +4,6 @@ import (
 	"os"
 	"regexp"
 	"strings"
-
-	"github.com/gochan-org/gochan/pkg/config"
 )
 
 var (
@@ -44,96 +42,3 @@ func RunSQLFile(path string) error {
 	}
 	return tx.Commit()
 }
-
-// TODO: get gochan-migration working so this doesn't have to sit here
-func tmpSqlAdjust() error {
-	// first update the crappy wordfilter table structure
-	var err error
-	var query string
-	switch gcdb.driver {
-	case "mysql":
-		query = `SELECT COUNT(*) FROM information_schema.TABLE_CONSTRAINTS
-		WHERE CONSTRAINT_NAME = 'wordfilters_board_id_fk'
-		AND TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'DBPREFIXwordfilters'`
-		var numConstraints int
-		if err = gcdb.QueryRowSQL(query,
-			interfaceSlice(),
-			interfaceSlice(&numConstraints)); err != nil {
-			return err
-		}
-		if numConstraints > 0 {
-			query = `ALTER TABLE DBPREFIXwordfilters DROP FOREIGN KEY wordfilters_board_id_fk`
-		} else {
-			query = ""
-		}
-		query = `SELECT COUNT(*) FROM information_schema.COLUMNS
-		WHERE TABLE_SCHEMA = DATABASE()
-		AND TABLE_NAME = 'DBPREFIXwordfilters'
-		AND COLUMN_NAME = 'board_dirs'`
-		var numColumns int
-		if err = gcdb.QueryRowSQL(query,
-			interfaceSlice(),
-			interfaceSlice(&numColumns)); err != nil {
-			return err
-		}
-		if numColumns == 0 {
-			query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
-			if _, err = ExecSQL(query); err != nil {
-				return err
-			}
-		}
-
-		// Yay, collation! Everybody loves MySQL's default collation!
-		criticalConfig := config.GetSystemCriticalConfig()
-		query = `ALTER DATABASE ` + criticalConfig.DBname + ` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci`
-		if _, err = gcdb.db.Exec(query); err != nil {
-			return err
-		}
-
-		query = `SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?`
-		rows, err := QuerySQL(query, criticalConfig.DBname)
-		if err != nil {
-			return err
-		}
-		defer rows.Close()
-		for rows.Next() {
-			var tableName string
-			err = rows.Scan(&tableName)
-			if err != nil {
-				return err
-			}
-			query = `ALTER TABLE ` + tableName + ` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`
-			if _, err = gcdb.db.Exec(query); err != nil {
-				return err
-			}
-		}
-		err = nil
-	case "postgres":
-		_, err = ExecSQL(`ALTER TABLE DBPREFIXwordfilters DROP CONSTRAINT IF EXISTS board_id_fk`)
-		if err != nil {
-			return err
-		}
-		query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN IF NOT EXISTS board_dirs varchar(255) DEFAULT '*'`
-		if _, err = ExecSQL(query); err != nil {
-			return err
-		}
-	case "sqlite3":
-		_, err = ExecSQL(`PRAGMA foreign_keys = ON`)
-		if err != nil {
-			return err
-		}
-		query = `SELECT COUNT(*) FROM PRAGMA_TABLE_INFO('DBPREFIXwordfilters') WHERE name = 'board_dirs'`
-		var numColumns int
-		if err = QueryRowSQL(query, interfaceSlice(), interfaceSlice(&numColumns)); err != nil {
-			return err
-		}
-		if numColumns == 0 {
-			query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
-			if _, err = ExecSQL(query); err != nil {
-				return err
-			}
-		}
-	}
-
-	return err
-}

pkg/gcsql/database.go

@@ -84,6 +84,26 @@ func (db *GCDB) ExecSQL(query string, values ...interface{}) (sql.Result, error)
 	return stmt.Exec(values...)
 }
 
+/*
+ExecTxSQL automatically escapes the given values and caches the statement
+Example:
+
+	tx, err := BeginTx()
+	// do error handling stuff
+	defer tx.Rollback()
+	var intVal int
+	var stringVal string
+	result, err := db.ExecTxSQL(tx, "INSERT INTO tablename (intval,stringval) VALUES(?,?)",
+		intVal, stringVal)
+*/
+func (db *GCDB) ExecTxSQL(tx *sql.Tx, query string, values ...interface{}) (sql.Result, error) {
+	stmt, err := db.PrepareSQL(query, tx)
+	if err != nil {
+		return nil, err
+	}
+	return stmt.Exec(values...)
+}
+
 /*
 Begin creates and returns a new SQL transaction using the GCDB. Note that it doesn't use gochan's
 database variables, e.g. DBPREFIX, DBNAME, etc so it should be used sparingly or with
@@ -123,6 +143,29 @@ func (db *GCDB) QueryRowSQL(query string, values, out []interface{}) error {
 	return stmt.QueryRow(values...).Scan(out...)
 }
 
+/*
+QueryRowTxSQL gets a row from the db with the values in values[] and fills the respective pointers in out[]
+Automatically escapes the given values and caches the query
+Example:
+
+	id := 32
+	var intVal int
+	var stringVal string
+	tx, err := BeginTx()
+	// do error handling stuff
+	defer tx.Rollback()
+	err = QueryRowTxSQL(tx, "SELECT intval,stringval FROM table WHERE id = ?",
+		[]interface{}{id},
+		[]interface{}{&intVal, &stringVal})
+*/
+func (db *GCDB) QueryRowTxSQL(tx *sql.Tx, query string, values, out []interface{}) error {
+	stmt, err := db.PrepareSQL(query, tx)
+	if err != nil {
+		return err
+	}
+	return stmt.QueryRow(values...).Scan(out...)
+}
+
 /*
 QuerySQL gets all rows from the db with the values in values[] and fills the respective pointers in out[]
 Automatically escapes the given values and caches the query

pkg/gcsql/provisioning.go

@@ -16,7 +16,7 @@ const (
 	DBUpToDate
 	DBModernButAhead
 
-	targetDatabaseVersion = 1
+	targetDatabaseVersion = 2
 )
 
 var (
@@ -108,7 +108,7 @@ func CheckAndInitializeDatabase(dbType string) error {
 	if err != nil {
 		return err
 	}
-	return tmpSqlAdjust()
+	return nil
 }
 
 func buildNewDatabase(dbType string) error {

pkg/gcsql/util.go

@@ -142,7 +142,7 @@ func QueryRowSQL(query string, values, out []interface{}) error {
 }
 
 /*
-QueryRowSQL gets a row from the db with the values in values[] and fills the respective pointers in out[]
+QueryRowTxSQL gets a row from the db with the values in values[] and fills the respective pointers in out[]
 Automatically escapes the given values and caches the query
 Example:
 
@@ -160,11 +160,7 @@ func QueryRowTxSQL(tx *sql.Tx, query string, values, out []interface{}) error {
 	if gcdb == nil {
 		return ErrNotConnected
 	}
-	stmt, err := PrepareSQL(query, tx)
-	if err != nil {
-		return err
-	}
-	return stmt.QueryRow(values...).Scan(out...)
+	return gcdb.QueryRowTxSQL(tx, query, values, out)
 }
 
 /*