gochan/gochan
1
0
Fork 0
mirror of https://github.com/Eggbertx/gochan.git synced 2025-08-02 02:36:24 -07:00
Code Releases Activity

Move database schema updating to gochan-migration

Browse source
This commit is contained in:
Eggbertx 2023-04-07 14:34:28 -07:00
parent 1da4330780
commit 17c28e5ebe
8 changed files with 260 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

1
build.py
View file

@ -39,6 +39,7 @@ release_files = (
)
GOCHAN_VERSION = "3.5.1"
DATABASE_VERSION = "2" # stored in DBNAME.DBPREFIXdatabase_version
PATH_NOTHING = -1
PATH_UNKNOWN = 0

2
cmd/gochan-migration/internal/common/sqlutil.go
View file

@ -24,7 +24,7 @@ func RunSQLFile(path string, db *gcsql.GCDB) error {
sqlArr := strings.Split(sqlStr, ";")
for _, statement := range sqlArr {
statement = strings.Trim(statement, " \n\r\t")
statement = strings.TrimSpace(statement)
if len(statement) > 0 {
if _, err = db.ExecSQL(statement); err != nil {
return err

184
cmd/gochan-migration/internal/gcupdate/gcupdate.go Normal file
View file

@ -0,0 +1,184 @@
package gcupdate
import (
"context"
"database/sql"
"fmt"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
"github.com/gochan-org/gochan/pkg/config"
"github.com/gochan-org/gochan/pkg/gcsql"
"github.com/gochan-org/gochan/pkg/gcutil"
)
const (
// if the database version is less than this, it is assumed to be out of date, and the schema needs to be adjusted
latestDatabaseVersion = 2
)
type GCDatabaseUpdater struct {
options *common.MigrationOptions
db *gcsql.GCDB
}
func (dbu *GCDatabaseUpdater) Init(options *common.MigrationOptions) error {
dbu.options = options
criticalCfg := config.GetSystemCriticalConfig()
var err error
dbu.db, err = gcsql.Open(
criticalCfg.DBhost, criticalCfg.DBtype, criticalCfg.DBname, criticalCfg.DBusername, criticalCfg.DBpassword,
criticalCfg.DBprefix,
)
return err
}
func (dbu *GCDatabaseUpdater) IsMigrated() (bool, error) {
var currentDatabaseVersion int
err := dbu.db.QueryRowSQL(`SELECT version FROM DBPREFIXdatabase_version WHERE component = 'gochan'`, nil,
[]any{&currentDatabaseVersion})
if err != nil {
return false, err
}
if currentDatabaseVersion == latestDatabaseVersion {
return true, nil
}
if currentDatabaseVersion > latestDatabaseVersion {
return false, fmt.Errorf("database layout is ahead of current version (%d), target version: %d",
currentDatabaseVersion, latestDatabaseVersion)
}
return false, nil
}
func (dbu *GCDatabaseUpdater) MigrateDB() (bool, error) {
migrated, err := dbu.IsMigrated()
if migrated || err != nil {
return migrated, err
}
var query string
criticalConfig := config.GetSystemCriticalConfig()
ctx := context.Background()
tx, err := dbu.db.BeginTx(ctx, &sql.TxOptions{
Isolation: 0,
ReadOnly: false,
})
if err != nil {
return false, err
}
defer tx.Rollback()
switch criticalConfig.DBtype {
case "mysql":
query = `SELECT COUNT(*) FROM information_schema.TABLE_CONSTRAINTS
WHERE CONSTRAINT_NAME = 'wordfilters_board_id_fk'
AND TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'DBPREFIXwordfilters'`
var numConstraints int
if err = dbu.db.QueryRowTxSQL(tx, query, nil, []any{&numConstraints}); err != nil {
return false, err
}
if numConstraints > 0 {
query = `ALTER TABLE DBPREFIXwordfilters DROP FOREIGN KEY wordfilters_board_id_fk`
} else {
query = ""
}
query = `SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'DBPREFIXwordfilters'
AND COLUMN_NAME = 'board_dirs'`
var numColumns int
if err = dbu.db.QueryRowTxSQL(tx, query, nil, []any{&numColumns}); err != nil {
return false, err
}
if numColumns == 0 {
query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
if _, err = gcsql.ExecTxSQL(tx, query); err != nil {
return false, err
}
}
// Yay, collation! Everybody loves MySQL's default collation!
query = `ALTER DATABASE ` + criticalConfig.DBname + ` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci`
if _, err = tx.Exec(query); err != nil {
return false, err
}
query = `SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?`
rows, err := dbu.db.QuerySQL(query, criticalConfig.DBname)
if err != nil {
return false, err
}
defer rows.Close()
for rows.Next() {
var tableName string
err = rows.Scan(&tableName)
if err != nil {
return false, err
}
query = `ALTER TABLE ` + tableName + ` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`
if _, err = tx.Exec(query); err != nil {
return false, err
}
}
err = nil
case "postgres":
_, err = gcsql.ExecSQL(`ALTER TABLE DBPREFIXwordfilters DROP CONSTRAINT IF EXISTS board_id_fk`)
if err != nil {
return false, err
}
query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN IF NOT EXISTS board_dirs varchar(255) DEFAULT '*'`
if _, err = dbu.db.ExecTxSQL(tx, query); err != nil {
return false, err
}
case "sqlite3":
_, err = gcsql.ExecSQL(`PRAGMA foreign_keys = ON`)
if err != nil {
return false, err
}
query = `SELECT COUNT(*) FROM PRAGMA_TABLE_INFO('DBPREFIXwordfilters') WHERE name = 'board_dirs'`
var numColumns int
if err = dbu.db.QueryRowSQL(query, nil, []any{&numColumns}); err != nil {
return false, err
}
if numColumns == 0 {
query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
if _, err = dbu.db.ExecTxSQL(tx, query); err != nil {
return false, err
}
}
}
query = `UPDATE DBPREFIXdatabase_version SET version = ? WHERE component = 'gochan'`
_, err = dbu.db.ExecTxSQL(tx, query, latestDatabaseVersion)
if err != nil {
return false, err
}
return false, tx.Commit()
}
func (dbu *GCDatabaseUpdater) MigrateBoards() error {
return gcutil.ErrNotImplemented
}
func (dbu *GCDatabaseUpdater) MigratePosts() error {
return gcutil.ErrNotImplemented
}
func (dbu *GCDatabaseUpdater) MigrateStaff(password string) error {
return gcutil.ErrNotImplemented
}
func (dbu *GCDatabaseUpdater) MigrateBans() error {
return gcutil.ErrNotImplemented
}
func (dbu *GCDatabaseUpdater) MigrateAnnouncements() error {
return gcutil.ErrNotImplemented
}
func (dbu *GCDatabaseUpdater) Close() error {
if dbu.db != nil {
return dbu.db.Close()
}
return nil
}

42
cmd/gochan-migration/main.go
View file

@ -3,8 +3,10 @@ package main
import (
"flag"
"log"
"os"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/gcupdate"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/pre2021"
"github.com/gochan-org/gochan/pkg/config"
@ -13,7 +15,7 @@ import (
const (
banner = `Welcome to the gochan database migration tool for gochan %s!
This migration tool is currently very unstable, and will likely go through
This migration tool is currently unstable, and will likely go through
several changes before it can be considered "stable", so make sure you check
the README and/or the -h command line flag before you use it.
@ -28,14 +30,17 @@ for the threads and board pages`
)
var (
versionStr string
versionStr string
dbVersionStr string
)
func main() {
var options common.MigrationOptions
var dirAction string
var updateDB bool
log.SetFlags(0)
flag.BoolVar(&updateDB, "updatedb", false, "If this is set, gochan-migrate will check, and if needed, update gochan's database schema")
flag.StringVar(&options.ChanType, "oldchan", "", "The imageboard we are migrating from (currently only pre2021 is supported, but more are coming")
flag.StringVar(&options.OldChanConfig, "oldconfig", "", "The path to the old chan's configuration file")
// flag.StringVar(&dirAction, "diraction", "", "Action taken on each board directory after it has been migrated. "+allowedDirActions)
@ -43,10 +48,12 @@ func main() {
config.InitConfig(versionStr)
if options.ChanType == "" || options.OldChanConfig == "" {
if !updateDB && (options.ChanType == "" || options.OldChanConfig == "") {
flag.PrintDefaults()
log.Fatal("Missing required oldchan value")
return
} else if updateDB {
options.ChanType = "gcupdate"
}
switch dirAction {
case "":
@ -64,6 +71,8 @@ func main() {
log.Printf(banner, versionStr)
var migrator common.DBMigrator
switch options.ChanType {
case "gcupdate":
migrator = &gcupdate.GCDatabaseUpdater{}
case "pre2021":
migrator = &pre2021.Pre2021Migrator{}
case "kusabax":
@ -77,18 +86,20 @@ func main() {
return
}
config.InitConfig(versionStr)
systemCritical := config.GetSystemCriticalConfig()
err := gcsql.ConnectToDB(
systemCritical.DBhost, systemCritical.DBtype, systemCritical.DBname,
systemCritical.DBusername, systemCritical.DBpassword, systemCritical.DBprefix)
if err != nil {
log.Fatalf("Failed to connect to the database: %s", err.Error())
var err error
if !updateDB {
systemCritical := config.GetSystemCriticalConfig()
err = gcsql.ConnectToDB(
systemCritical.DBhost, systemCritical.DBtype, systemCritical.DBname,
systemCritical.DBusername, systemCritical.DBpassword, systemCritical.DBprefix)
if err != nil {
log.Fatalf("Failed to connect to the database: %s", err.Error())
}
if err = gcsql.CheckAndInitializeDatabase(systemCritical.DBtype); err != nil {
log.Fatalf("Failed to initialize the database: %s", err.Error())
}
defer gcsql.Close()
}
if err = gcsql.CheckAndInitializeDatabase(systemCritical.DBtype); err != nil {
log.Fatalf("Failed to initialize the database: %s", err.Error())
}
defer gcsql.Close()
if err = migrator.Init(&options); err != nil {
log.Fatalf("Unable to initialize %s migrator: %s\n",
@ -102,7 +113,8 @@ func main() {
log.Fatalln("Error migrating database:", err.Error())
}
if migrated {
log.Fatalf("Database is already migrated")
log.Println("Database is already migrated")
os.Exit(0)
}
log.Println(migrateCompleteTxt)
}

95
pkg/gcsql/connect.go
View file

@ -4,8 +4,6 @@ import (
"os"
"regexp"
"strings"
"github.com/gochan-org/gochan/pkg/config"
)
var (
@ -44,96 +42,3 @@ func RunSQLFile(path string) error {
}
return tx.Commit()
}
// TODO: get gochan-migration working so this doesn't have to sit here
func tmpSqlAdjust() error {
// first update the crappy wordfilter table structure
var err error
var query string
switch gcdb.driver {
case "mysql":
query = `SELECT COUNT(*) FROM information_schema.TABLE_CONSTRAINTS
WHERE CONSTRAINT_NAME = 'wordfilters_board_id_fk'
AND TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'DBPREFIXwordfilters'`
var numConstraints int
if err = gcdb.QueryRowSQL(query,
interfaceSlice(),
interfaceSlice(&numConstraints)); err != nil {
return err
}
if numConstraints > 0 {
query = `ALTER TABLE DBPREFIXwordfilters DROP FOREIGN KEY wordfilters_board_id_fk`
} else {
query = ""
}
query = `SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'DBPREFIXwordfilters'
AND COLUMN_NAME = 'board_dirs'`
var numColumns int
if err = gcdb.QueryRowSQL(query,
interfaceSlice(),
interfaceSlice(&numColumns)); err != nil {
return err
}
if numColumns == 0 {
query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
if _, err = ExecSQL(query); err != nil {
return err
}
}
// Yay, collation! Everybody loves MySQL's default collation!
criticalConfig := config.GetSystemCriticalConfig()
query = `ALTER DATABASE ` + criticalConfig.DBname + ` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci`
if _, err = gcdb.db.Exec(query); err != nil {
return err
}
query = `SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?`
rows, err := QuerySQL(query, criticalConfig.DBname)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var tableName string
err = rows.Scan(&tableName)
if err != nil {
return err
}
query = `ALTER TABLE ` + tableName + ` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`
if _, err = gcdb.db.Exec(query); err != nil {
return err
}
}
err = nil
case "postgres":
_, err = ExecSQL(`ALTER TABLE DBPREFIXwordfilters DROP CONSTRAINT IF EXISTS board_id_fk`)
if err != nil {
return err
}
query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN IF NOT EXISTS board_dirs varchar(255) DEFAULT '*'`
if _, err = ExecSQL(query); err != nil {
return err
}
case "sqlite3":
_, err = ExecSQL(`PRAGMA foreign_keys = ON`)
if err != nil {
return err
}
query = `SELECT COUNT(*) FROM PRAGMA_TABLE_INFO('DBPREFIXwordfilters') WHERE name = 'board_dirs'`
var numColumns int
if err = QueryRowSQL(query, interfaceSlice(), interfaceSlice(&numColumns)); err != nil {
return err
}
if numColumns == 0 {
query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
if _, err = ExecSQL(query); err != nil {
return err
}
}
}
return err
}

43
pkg/gcsql/database.go
View file

@ -84,6 +84,26 @@ func (db *GCDB) ExecSQL(query string, values ...interface{}) (sql.Result, error)
return stmt.Exec(values...)
}
/*
ExecTxSQL automatically escapes the given values and caches the statement
Example:
tx, err := BeginTx()
// do error handling stuff
defer tx.Rollback()
var intVal int
var stringVal string
result, err := db.ExecTxSQL(tx, "INSERT INTO tablename (intval,stringval) VALUES(?,?)",
intVal, stringVal)
*/
func (db *GCDB) ExecTxSQL(tx *sql.Tx, query string, values ...interface{}) (sql.Result, error) {
stmt, err := db.PrepareSQL(query, tx)
if err != nil {
return nil, err
}
return stmt.Exec(values...)
}
/*
Begin creates and returns a new SQL transaction using the GCDB. Note that it doesn't use gochan's
database variables, e.g. DBPREFIX, DBNAME, etc so it should be used sparingly or with
@ -123,6 +143,29 @@ func (db *GCDB) QueryRowSQL(query string, values, out []interface{}) error {
return stmt.QueryRow(values...).Scan(out...)
}
/*
QueryRowTxSQL gets a row from the db with the values in values[] and fills the respective pointers in out[]
Automatically escapes the given values and caches the query
Example:
id := 32
var intVal int
var stringVal string
tx, err := BeginTx()
// do error handling stuff
defer tx.Rollback()
err = QueryRowTxSQL(tx, "SELECT intval,stringval FROM table WHERE id = ?",
[]interface{}{id},
[]interface{}{&intVal, &stringVal})
*/
func (db *GCDB) QueryRowTxSQL(tx *sql.Tx, query string, values, out []interface{}) error {
stmt, err := db.PrepareSQL(query, tx)
if err != nil {
return err
}
return stmt.QueryRow(values...).Scan(out...)
}
/*
QuerySQL gets all rows from the db with the values in values[] and fills the respective pointers in out[]
Automatically escapes the given values and caches the query

4
pkg/gcsql/provisioning.go
View file

@ -16,7 +16,7 @@ const (
DBUpToDate
DBModernButAhead
targetDatabaseVersion = 1
targetDatabaseVersion = 2
)
var (
@ -108,7 +108,7 @@ func CheckAndInitializeDatabase(dbType string) error {
if err != nil {
return err
}
return tmpSqlAdjust()
return nil
}
func buildNewDatabase(dbType string) error {

8
pkg/gcsql/util.go
View file

@ -142,7 +142,7 @@ func QueryRowSQL(query string, values, out []interface{}) error {
}
/*
QueryRowSQL gets a row from the db with the values in values[] and fills the respective pointers in out[]
QueryRowTxSQL gets a row from the db with the values in values[] and fills the respective pointers in out[]
Automatically escapes the given values and caches the query
Example:
@ -160,11 +160,7 @@ func QueryRowTxSQL(tx *sql.Tx, query string, values, out []interface{}) error {
if gcdb == nil {
return ErrNotConnected
}
stmt, err := PrepareSQL(query, tx)
if err != nil {
return err
}
return stmt.QueryRow(values...).Scan(out...)
return gcdb.QueryRowTxSQL(tx, query, values, out)
}
/*