gochan/gochan
1
0
Fork 0
mirror of https://github.com/Eggbertx/gochan.git synced 2025-08-02 23:26:23 -07:00
Code Releases Activity

Use more context/timeout queries in gochan-migrate, start using zerolog

Browse source
This commit is contained in:
Eggbertx 2024-10-13 13:43:08 -07:00
parent 9d003d89a3
commit eb06047055
11 changed files with 307 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

50
cmd/gochan-migration/internal/common/filters.go
View file

@ -4,6 +4,7 @@ import (
"context" "context"
"database/sql" "database/sql"
"errors" "errors"
"fmt"
"os" "os"
"strings" "strings"
"time" "time"
@ -39,7 +40,15 @@ func AddFilterTables(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig
// MigrateFileBans migrates file checksum and image fingerprint bans to the filter table // MigrateFileBans migrates file checksum and image fingerprint bans to the filter table
func MigrateFileBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error { func MigrateFileBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,checksum,fingerprinter,ban_ip,ban_ip_message FROM DBPREFIXfile_ban`) fileBanTableExists, err := TableExists(ctx, db, tx, "DBPREFIXfilename_ban", cfg)
if err != nil {
return err
}
if !fileBanTableExists {
// no filename bans to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_id,staff_id,staff_note,issued_at,checksum,fingerprinter,ban_ip,ban_ip_message FROM DBPREFIXfile_ban`)
if err != nil { if err != nil {
return err return err
} }
@ -106,10 +115,19 @@ func MigrateFileBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *confi
// MigrateFilenameBans migrates filename bans to the filter table // MigrateFilenameBans migrates filename bans to the filter table
func MigrateFilenameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error { func MigrateFilenameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,filename,is_regex FROM DBPREFIXfilename_ban`) filenameBanTableExists, err := TableExists(ctx, db, tx, "DBPREFIXfilename_ban", cfg)
if err != nil { if err != nil {
return err return err
} }
if !filenameBanTableExists {
// no filename bans to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_id,staff_id,staff_note,issued_at,filename,is_regex FROM DBPREFIXfilename_ban`)
if err != nil {
fmt.Println("query error")
return err
}
defer rows.Close() defer rows.Close()
var fnBanBoardID *int var fnBanBoardID *int
@ -162,10 +180,19 @@ func MigrateFilenameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *c
// MigrateUsernameBans migrates poster name bans to the filter table // MigrateUsernameBans migrates poster name bans to the filter table
func MigrateUsernameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error { func MigrateUsernameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,username,is_regex FROM DBPREFIXusername_ban`) usernameBanTableExists, err := TableExists(ctx, db, tx, "DBPREFIXusername_ban", cfg)
if err != nil { if err != nil {
return err return err
} }
if !usernameBanTableExists {
// no name bans to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_id,staff_id,staff_note,issued_at,username,is_regex FROM DBPREFIXusername_ban`)
if err != nil {
fmt.Println("MigrateUsernameBans rows error")
return err
}
defer rows.Close() defer rows.Close()
var unBanBoardID *int var unBanBoardID *int
@ -184,7 +211,7 @@ func MigrateUsernameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *c
} }
if _, err = db.ExecContextSQL(ctx, tx, if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilters(staff_id, staff_note, issued_at, match_action, match_detail, is_active) VALUES(?,?,?,?,?)`, `INSERT INTO DBPREFIXfilters(staff_id, staff_note, issued_at, match_action, match_detail, is_active) VALUES(?,?,?,?,?,?)`,
unBanStaffID, unBanStaffNote, unBanIssuedAt, "reject", "", true, unBanStaffID, unBanStaffNote, unBanIssuedAt, "reject", "", true,
); err != nil { ); err != nil {
return err return err
@ -213,16 +240,24 @@ func MigrateUsernameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *c
return err return err
} }
} }
return rows.Close() return rows.Close()
} }
// MigrateWordfilters migrates pre-filter wordfilters to the filter table // MigrateWordfilters migrates pre-filter wordfilters to the filter table
func MigrateWordfilters(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error { func MigrateWordfilters(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_dirs, staff_id, staff_note, issued_at, search, is_regex, change_to FROM DBPREFIXwordfilters`) wordfiltersTableExists, err := TableExists(ctx, db, tx, "DBPREFIXwordfilters", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if !wordfiltersTableExists {
// no wordfilters to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_dirs, staff_id, staff_note, issued_at, search, is_regex, change_to FROM DBPREFIXwordfilters`)
if err != nil {
fmt.Println("MigrateWordfilters rows error")
return err
}
defer rows.Close() defer rows.Close()
var boardDirsPtr *string var boardDirsPtr *string
@ -281,7 +316,8 @@ func MigrateWordfilters(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConf
} }
if _, err = db.ExecContextSQL(ctx, tx, if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilter_conditions(filter_id, match_mode, search, field) VALUES(?,?,?,'body')`, filterID, matchMode, search, `INSERT INTO DBPREFIXfilter_conditions(filter_id, match_mode, search, field) VALUES(?,?,?,'body')`,
filterID, matchMode, search,
); err != nil { ); err != nil {
return err return err
} }

3
cmd/gochan-migration/internal/common/handler.go
View file

@ -57,7 +57,8 @@ type DBMigrator interface {
// will exit // will exit
IsMigrated() (bool, error) IsMigrated() (bool, error)
// MigrateDB migrates the imageboard data (posts, boards, etc) to the new database // MigrateDB migrates the imageboard data (posts, boards, etc) to the new database. It is
// assumed that MigrateDB will handle logging any errors that occur during the migration
MigrateDB() (bool, error) MigrateDB() (bool, error)
// MigrateBoards gets info about the old boards in the board table and inserts each one // MigrateBoards gets info about the old boards in the board table and inserts each one

67
cmd/gochan-migration/internal/common/logger.go Normal file
View file

@ -0,0 +1,67 @@
package common
import (
"io"
"io/fs"
"os"
"path"
"github.com/gochan-org/gochan/pkg/config"
"github.com/gochan-org/gochan/pkg/gcutil"
"github.com/rs/zerolog"
)
const (
logFlags = os.O_CREATE | os.O_APPEND | os.O_WRONLY
logFileMode fs.FileMode = 0644
)
var (
migrationLogFile *os.File
migrationLog zerolog.Logger
)
func InitMigrationLog() (err error) {
if migrationLogFile != nil {
// Migration log already initialized
return nil
}
logPath := path.Join(config.GetSystemCriticalConfig().LogDir, "migration.log")
migrationLogFile, err = os.OpenFile(logPath, logFlags, logFileMode)
if err != nil {
return err
}
var writer io.Writer
cw := zerolog.NewConsoleWriter()
cw.NoColor = !gcutil.RunningInTerminal()
writer = zerolog.MultiLevelWriter(migrationLogFile, cw)
migrationLog = zerolog.New(writer).With().Timestamp().Logger()
return nil
}
func Logger() *zerolog.Logger {
return &migrationLog
}
func LogInfo() *zerolog.Event {
return migrationLog.Info()
}
func LogWarning() *zerolog.Event {
return migrationLog.Warn()
}
func LogError() *zerolog.Event {
return migrationLog.Error()
}
func LogFatal() *zerolog.Event {
return migrationLog.Fatal()
}
func CloseLog() error {
if migrationLogFile == nil {
return nil
}
return migrationLogFile.Close()
}

22
cmd/gochan-migration/internal/common/sqlutil.go
View file

@ -1,6 +1,7 @@
package common package common
import ( import (
"context"
"database/sql" "database/sql"
"errors" "errors"
"fmt" "fmt"
@ -20,7 +21,7 @@ var (
// ColumnType returns a string representation of the column's data type. It does not return an error // ColumnType returns a string representation of the column's data type. It does not return an error
// if the column does not exist, instead returning an empty string. // if the column does not exist, instead returning an empty string.
func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string, sqlConfig *config.SQLConfig) (string, error) { func ColumnType(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string, sqlConfig *config.SQLConfig) (string, error) {
var query string var query string
var dataType string var dataType string
var err error var err error
@ -32,7 +33,7 @@ func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string,
query = `SELECT DATA_TYPE FROM information_schema.COLUMNS query = `SELECT DATA_TYPE FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1` WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1`
params = []any{dbName, tableName, columnName} params = []any{dbName, tableName, columnName}
case "postgresql": case "postgres", "postgresql":
query = `SELECT data_type FROM information_schema.columns query = `SELECT data_type FROM information_schema.columns
WHERE (table_schema = ? OR table_schema = 'public') WHERE (table_schema = ? OR table_schema = 'public')
AND table_name = ? AND column_name = ? LIMIT 1` AND table_name = ? AND column_name = ? LIMIT 1`
@ -43,7 +44,7 @@ func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string,
default: default:
return "", gcsql.ErrUnsupportedDB return "", gcsql.ErrUnsupportedDB
} }
err = db.QueryRowTxSQL(tx, query, params, []any{&dataType}) err = db.QueryRowContextSQL(ctx, tx, query, params, []any{&dataType})
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return "", nil return "", nil
} }
@ -51,26 +52,21 @@ func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string,
} }
// TableExists returns true if the given table exists in the given database, and an error if one occured // TableExists returns true if the given table exists in the given database, and an error if one occured
func TableExists(db *gcsql.GCDB, tx *sql.Tx, tableName string, sqlConfig *config.SQLConfig) (bool, error) { func TableExists(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, tableName string, sqlConfig *config.SQLConfig) (bool, error) {
tableName = strings.ReplaceAll(tableName, "DBPREFIX", sqlConfig.DBprefix) tableName = strings.ReplaceAll(tableName, "DBPREFIX", sqlConfig.DBprefix)
dbName := sqlConfig.DBname
var query string var query string
var params []any
switch sqlConfig.DBtype { switch sqlConfig.DBtype {
case "mysql": case "mysql":
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?` query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?`
params = []any{dbName, tableName} case "postgres", "postgresql":
case "postgresql": query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE table_catalog = CURRENT_DATABASE() AND table_name = ?`
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE table_catalog = ? AND table_name = ?`
params = []any{dbName, tableName}
case "sqlite3": case "sqlite3":
query = `SELECT COUNT(*) FROM sqlite_master WHERE name = ? AND type = 'table'` query = `SELECT COUNT(*) FROM sqlite_master WHERE name = ? AND type = 'table'`
params = []any{tableName}
default: default:
return false, gcsql.ErrUnsupportedDB return false, gcsql.ErrUnsupportedDB
} }
var count int var count int
err := db.QueryRowTxSQL(tx, query, params, []any{&count}) err := db.QueryRowContextSQL(ctx, tx, query, []any{tableName}, []any{&count})
return count == 1, err return count == 1, err
} }

36
cmd/gochan-migration/internal/gcupdate/gcupdate.go
View file

@ -4,6 +4,7 @@ import (
"context" "context"
"database/sql" "database/sql"
"fmt" "fmt"
"time"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
"github.com/gochan-org/gochan/pkg/config" "github.com/gochan-org/gochan/pkg/config"
@ -51,7 +52,8 @@ func (dbu *GCDatabaseUpdater) MigrateDB() (bool, error) {
} }
sqlConfig := config.GetSQLConfig() sqlConfig := config.GetSQLConfig()
ctx := context.Background() ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
tx, err := dbu.db.BeginTx(ctx, &sql.TxOptions{ tx, err := dbu.db.BeginTx(ctx, &sql.TxOptions{
Isolation: 0, Isolation: 0,
ReadOnly: false, ReadOnly: false,
@ -63,46 +65,56 @@ func (dbu *GCDatabaseUpdater) MigrateDB() (bool, error) {
switch sqlConfig.DBtype { switch sqlConfig.DBtype {
case "mysql": case "mysql":
err = updateMysqlDB(dbu.db, tx, &sqlConfig) err = updateMysqlDB(ctx, dbu.db, tx, &sqlConfig)
case "postgres": case "postgres":
err = updatePostgresDB(dbu.db, tx, &sqlConfig) err = updatePostgresDB(ctx, dbu.db, tx, &sqlConfig)
case "sqlite3": case "sqlite3":
err = updateSqliteDB(dbu.db, tx, &sqlConfig) err = updateSqliteDB(ctx, dbu.db, tx, &sqlConfig)
} }
if err != nil { if err != nil {
return false, err return false, err
} }
filterTableExists, err := common.TableExists(dbu.db, nil, "DBPREFIXfilters", &sqlConfig) // commit the transaction and start a new one (to avoid deadlocks)
if err = tx.Commit(); err != nil {
return false, err
}
if err = ctx.Err(); err != nil {
return false, err
}
filterTableExists, err := common.TableExists(ctx, dbu.db, nil, "DBPREFIXfilters", &sqlConfig)
if err != nil { if err != nil {
return false, err return false, err
} }
if !filterTableExists { if !filterTableExists {
// DBPREFIXfilters not found, create it and migrate data from DBPREFIXfile_bans, DBPREFIXfilename_bans, and DBPREFIXusername_bans, // DBPREFIXfilters not found, create it and migrate data from DBPREFIXfile_bans, DBPREFIXfilename_bans, and DBPREFIXusername_bans,
if err = common.AddFilterTables(ctx, dbu.db, tx, &sqlConfig); err != nil { if err = common.AddFilterTables(ctx, dbu.db, nil, &sqlConfig); err != nil {
return false, err return false, err
} }
if err = common.MigrateFileBans(ctx, dbu.db, tx, &sqlConfig); err != nil { if err = common.MigrateFileBans(ctx, dbu.db, nil, &sqlConfig); err != nil {
return false, err return false, err
} }
if err = common.MigrateFilenameBans(ctx, dbu.db, tx, &sqlConfig); err != nil { if err = common.MigrateFilenameBans(ctx, dbu.db, nil, &sqlConfig); err != nil {
return false, err return false, err
} }
if err = common.MigrateUsernameBans(ctx, dbu.db, tx, &sqlConfig); err != nil { if err = common.MigrateUsernameBans(ctx, dbu.db, nil, &sqlConfig); err != nil {
return false, err return false, err
} }
if err = common.MigrateWordfilters(ctx, dbu.db, tx, &sqlConfig); err != nil { if err = common.MigrateWordfilters(ctx, dbu.db, nil, &sqlConfig); err != nil {
return false, err return false, err
} }
} }
query := `UPDATE DBPREFIXdatabase_version SET version = ? WHERE component = 'gochan'` query := `UPDATE DBPREFIXdatabase_version SET version = ? WHERE component = 'gochan'`
_, err = dbu.db.ExecTxSQL(tx, query, dbu.TargetDBVer) _, err = dbu.db.ExecContextSQL(ctx, nil, query, dbu.TargetDBVer)
if err != nil { if err != nil {
return false, err return false, err
} }
return false, tx.Commit() // return false, tx.Commit()
return false, nil
} }
func (*GCDatabaseUpdater) MigrateBoards() error { func (*GCDatabaseUpdater) MigrateBoards() error {

93
cmd/gochan-migration/internal/gcupdate/updatemysql.go
View file

@ -1,6 +1,7 @@
package gcupdate package gcupdate
import ( import (
"context"
"database/sql" "database/sql"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
@ -9,15 +10,25 @@ import (
"github.com/gochan-org/gochan/pkg/gcutil" "github.com/gochan-org/gochan/pkg/gcutil"
) )
func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error { func updateMysqlDB(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
var numConstraints int var query string
var err error var dataType string
dbName := sqlConfig.DBname dbName := sqlConfig.DBname
query := `SELECT COUNT(*) FROM information_schema.TABLE_CONSTRAINTS
wordfiltersTableExists, err := common.TableExists(ctx, db, tx, "DBPREFIXwordfilters", sqlConfig)
if err != nil {
return err
}
if wordfiltersTableExists {
// wordfilters table is going to be migrated by the end of the update, but we want to make sure its legacy data is migrated first
// so it can be properly merged into the filter table
var numConstraints int
query = `SELECT COUNT(*) FROM information_schema.TABLE_CONSTRAINTS
WHERE CONSTRAINT_NAME = 'wordfilters_board_id_fk' WHERE CONSTRAINT_NAME = 'wordfilters_board_id_fk'
AND TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'DBPREFIXwordfilters'` AND TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'DBPREFIXwordfilters'`
if err = db.QueryRowTxSQL(tx, query, nil, []any{&numConstraints}); err != nil { if err = db.QueryRowContextSQL(ctx, tx, query, nil, []any{&numConstraints}); err != nil {
return err return err
} }
if numConstraints > 0 { if numConstraints > 0 {
@ -25,25 +36,26 @@ func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) erro
} else { } else {
query = "" query = ""
} }
dataType, err := common.ColumnType(db, tx, "board_dirs", "DBPREFIXwordfilters", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "board_dirs", "DBPREFIXwordfilters", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'` query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
}
// Yay, collation! Everybody loves MySQL's default collation! // Yay, collation! Everybody loves MySQL's default collation!
query = `ALTER DATABASE ` + dbName + ` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci` query = `ALTER DATABASE ` + dbName + ` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci`
if _, err = tx.Exec(query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
query = `SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?` query = `SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?`
rows, err := db.QuerySQL(query, dbName) rows, err := db.QueryContextSQL(ctx, tx, query, dbName)
if err != nil { if err != nil {
return err return err
} }
@ -57,14 +69,15 @@ func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) erro
return err return err
} }
query = `ALTER TABLE ` + tableName + ` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci` query = `ALTER TABLE ` + tableName + ` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`
if _, err = tx.Exec(query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
if err = rows.Close(); err != nil { if err = rows.Close(); err != nil {
return err return err
} }
dataType, err = common.ColumnType(db, tx, "ip", "DBPREFIXip_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "ip", "DBPREFIXip_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
@ -73,11 +86,11 @@ func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) erro
query = `ALTER TABLE DBPREFIXip_ban query = `ALTER TABLE DBPREFIXip_ban
ADD COLUMN IF NOT EXISTS range_start VARBINARY(16) NOT NULL, ADD COLUMN IF NOT EXISTS range_start VARBINARY(16) NOT NULL,
ADD COLUMN IF NOT EXISTS range_end VARBINARY(16) NOT NULL` ADD COLUMN IF NOT EXISTS range_end VARBINARY(16) NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
// convert ban IP string to IP range // convert ban IP string to IP range
if rows, err = db.QuerySQL(`SELECT id, ip FROM DBPREFIXip_ban`); err != nil { if rows, err = db.QueryContextSQL(ctx, tx, `SELECT id, ip FROM DBPREFIXip_ban`); err != nil {
return err return err
} }
var rangeStart, rangeEnd string var rangeStart, rangeEnd string
@ -92,11 +105,11 @@ func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) erro
} }
query = `UPDATE DBPREFIXip_ban query = `UPDATE DBPREFIXip_ban
SET range_start = INET6_ATON(?), range_end = INET6_ATON(?) WHERE id = ?` SET range_start = INET6_ATON(?), range_end = INET6_ATON(?) WHERE id = ?`
if _, err = db.ExecTxSQL(tx, query, rangeStart, rangeEnd, id); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query, rangeStart, rangeEnd, id); err != nil {
return err return err
} }
query = `ALTER TABLE DBPREFIXip_ban DROP COLUMN IF EXISTS ip` query = `ALTER TABLE DBPREFIXip_ban DROP COLUMN IF EXISTS ip`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
@ -106,124 +119,132 @@ func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) erro
} }
// Convert DBPREFIXposts.ip to from varchar to varbinary // Convert DBPREFIXposts.ip to from varchar to varbinary
dataType, err = common.ColumnType(db, tx, "ip", "DBPREFIXposts", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "ip", "DBPREFIXposts", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if common.IsStringType(dataType) { if common.IsStringType(dataType) {
// rename `ip` to a temporary column to then be removed // rename `ip` to a temporary column to then be removed
query = `ALTER TABLE DBPREFIXposts CHANGE ip ip_str varchar(45)` query = `ALTER TABLE DBPREFIXposts CHANGE ip ip_str varchar(45)`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
query = `ALTER TABLE DBPREFIXposts query = `ALTER TABLE DBPREFIXposts
ADD COLUMN IF NOT EXISTS ip VARBINARY(16) NOT NULL` ADD COLUMN IF NOT EXISTS ip VARBINARY(16) NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
// convert post IP VARCHAR(45) to VARBINARY(16) // convert post IP VARCHAR(45) to VARBINARY(16)
query = `UPDATE DBPREFIXposts SET ip = INET6_ATON(ip_str)` query = `UPDATE DBPREFIXposts SET ip = INET6_ATON(ip_str)`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
query = `ALTER TABLE DBPREFIXposts DROP COLUMN IF EXISTS ip_str` query = `ALTER TABLE DBPREFIXposts DROP COLUMN IF EXISTS ip_str`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// Convert DBPREFIXreports.ip to from varchar to varbinary // Convert DBPREFIXreports.ip to from varchar to varbinary
dataType, err = common.ColumnType(db, tx, "ip", "DBPREFIXreports", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "ip", "DBPREFIXreports", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if common.IsStringType(dataType) { if common.IsStringType(dataType) {
// rename `ip` to a temporary column to then be removed // rename `ip` to a temporary column to then be removed
query = `ALTER TABLE DBPREFIXreports CHANGE ip ip_str varchar(45)` query = `ALTER TABLE DBPREFIXreports CHANGE ip ip_str varchar(45)`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
query = `ALTER TABLE DBPREFIXreports query = `ALTER TABLE DBPREFIXreports
ADD COLUMN IF NOT EXISTS ip VARBINARY(16) NOT NULL` ADD COLUMN IF NOT EXISTS ip VARBINARY(16) NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
// convert report IP VARCHAR(45) to VARBINARY(16) // convert report IP VARCHAR(45) to VARBINARY(16)
query = `UPDATE DBPREFIXreports SET ip = INET6_ATON(ip_str)` query = `UPDATE DBPREFIXreports SET ip = INET6_ATON(ip_str)`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
query = `ALTER TABLE DBPREFIXreports DROP COLUMN IF EXISTS ip_str` query = `ALTER TABLE DBPREFIXreports DROP COLUMN IF EXISTS ip_str`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add flag column to DBPREFIXposts // add flag column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "flag", "DBPREFIXposts", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "flag", "DBPREFIXposts", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''` query = `ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add country column to DBPREFIXposts // add country column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "country", "DBPREFIXposts", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "country", "DBPREFIXposts", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''` query = `ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
fileBanTableExists, err := common.TableExists(ctx, db, tx, "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if fileBanTableExists {
// file ban table is going to be migrated by the end of the update, but we want to make sure its legacy data is migrated first
// so it can be properly merged into the filter table
// add fingerprinter column to DBPREFIXfile_ban // add fingerprinter column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN fingerprinter VARCHAR(64)` query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN fingerprinter VARCHAR(64)`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add ban_ip column to DBPREFIXfile_ban // add ban_ip column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip BOOL NOT NULL` query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip BOOL NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add ban_ip_message column to DBPREFIXfile_ban // add ban_ip_message column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip_message TEXT` query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip_message TEXT`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
}
return nil return nil
} }

44
cmd/gochan-migration/internal/gcupdate/updatepostgres.go
View file

@ -1,6 +1,7 @@
package gcupdate package gcupdate
import ( import (
"context"
"database/sql" "database/sql"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
@ -8,27 +9,28 @@ import (
"github.com/gochan-org/gochan/pkg/gcsql" "github.com/gochan-org/gochan/pkg/gcsql"
) )
func updatePostgresDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error { func updatePostgresDB(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
query := `ALTER TABLE DBPREFIXwordfilters query := `ALTER TABLE DBPREFIXwordfilters
DROP CONSTRAINT IF EXISTS board_id_fk` DROP CONSTRAINT IF EXISTS board_id_fk`
_, err := db.ExecSQL(query) _, err := db.ExecContextSQL(ctx, tx, query)
if err != nil { if err != nil {
return err return err
} }
query = `ALTER TABLE DBPREFIXwordfilters query = `ALTER TABLE DBPREFIXwordfilters
ADD COLUMN IF NOT EXISTS board_dirs varchar(255) DEFAULT '*'` ADD COLUMN IF NOT EXISTS board_dirs varchar(255) DEFAULT '*'`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
dataType, err := common.ColumnType(db, tx, "ip", "DBPREFIXposts", sqlConfig) dataType, err := common.ColumnType(ctx, db, tx, "ip", "DBPREFIXposts", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if common.IsStringType(dataType) { if common.IsStringType(dataType) {
// change ip column to temporary ip_str // change ip column to temporary ip_str
query = `ALTER TABLE DBPREFIXposts RENAME COLUMN ip TO ip_str,` query = `ALTER TABLE DBPREFIXposts RENAME COLUMN ip TO ip_str,`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
@ -36,22 +38,22 @@ func updatePostgresDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) e
// because it is non-nil // because it is non-nil
query = `ALTER TABLE DBPREFIXposts query = `ALTER TABLE DBPREFIXposts
ADD COLUMN IF NOT EXISTS ip INET NOT NULL DEFAULT '127.0.0.1'` ADD COLUMN IF NOT EXISTS ip INET NOT NULL DEFAULT '127.0.0.1'`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
query = `UPDATE TABLE DBPREFIXposts SET ip = ip_str` query = `UPDATE TABLE DBPREFIXposts SET ip = ip_str`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
query = `ALTER TABLE DBPREFIXposts DROP COLUMN ip_str` query = `ALTER TABLE DBPREFIXposts DROP COLUMN ip_str`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
dataType, err = common.ColumnType(db, tx, "ip", "DBPREFIXip_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "ip", "DBPREFIXip_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
@ -59,72 +61,72 @@ func updatePostgresDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) e
query = `ALTER TABLE DBPREFIXip_ban query = `ALTER TABLE DBPREFIXip_ban
ADD COLUMN IF NOT EXISTS range_start INET NOT NULL, ADD COLUMN IF NOT EXISTS range_start INET NOT NULL,
ADD COLUMN IF NOT EXISTS range_end INET NOT NULL` ADD COLUMN IF NOT EXISTS range_end INET NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
query = `UPDATE DBPREFIXip_ban SET range_start = ip::INET, SET range_end = ip::INET` query = `UPDATE DBPREFIXip_ban SET range_start = ip::INET, SET range_end = ip::INET`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add flag column to DBPREFIXposts // add flag column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "flag", "DBPREFIXposts", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "flag", "DBPREFIXposts", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''` query = `ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add country column to DBPREFIXposts // add country column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "country", "DBPREFIXposts", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "country", "DBPREFIXposts", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''` query = `ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add fingerprinter column to DBPREFIXfile_ban // add fingerprinter column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN fingerprinter VARCHAR(64)` query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN fingerprinter VARCHAR(64)`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add ban_ip column to DBPREFIXfile_ban // add ban_ip column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip BOOL NOT NULL` query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip BOOL NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add ban_ip_message column to DBPREFIXfile_ban // add ban_ip_message column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip_message TEXT` query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip_message TEXT`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }

33
cmd/gochan-migration/internal/gcupdate/updatesqlite3.go
View file

@ -1,6 +1,7 @@
package gcupdate package gcupdate
import ( import (
"context"
"database/sql" "database/sql"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
@ -8,13 +9,15 @@ import (
"github.com/gochan-org/gochan/pkg/gcsql" "github.com/gochan-org/gochan/pkg/gcsql"
) )
func updateSqliteDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error { func updateSqliteDB(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
var query string var query string
_, err := db.ExecSQL(`PRAGMA foreign_keys = ON`)
_, err := db.ExecContextSQL(ctx, tx, `PRAGMA foreign_keys = ON`)
if err != nil { if err != nil {
return err return err
} }
dataType, err := common.ColumnType(db, tx, "DBPREFIXwordfilters", "board_dirs", sqlConfig)
dataType, err := common.ColumnType(ctx, db, tx, "DBPREFIXwordfilters", "board_dirs", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
@ -26,7 +29,7 @@ func updateSqliteDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) err
} }
// Add range_start column to DBPREFIXIp_ban if it doesn't exist // Add range_start column to DBPREFIXIp_ban if it doesn't exist
dataType, err = common.ColumnType(db, tx, "DBPREFIXip_ban", "range_start", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "DBPREFIXip_ban", "range_start", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
@ -38,7 +41,7 @@ func updateSqliteDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) err
} }
// Add range_start column if it doesn't exist // Add range_start column if it doesn't exist
dataType, err = common.ColumnType(db, tx, "DBPREFIXip_ban", "range_end", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "DBPREFIXip_ban", "range_end", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
@ -50,61 +53,61 @@ func updateSqliteDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) err
} }
// add flag column to DBPREFIXposts // add flag column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "flag", "DBPREFIXposts", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "flag", "DBPREFIXposts", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''` query = `ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add country column to DBPREFIXposts // add country column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "country", "DBPREFIXposts", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "country", "DBPREFIXposts", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''` query = `ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add fingerprinter column to DBPREFIXfile_ban // add fingerprinter column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN fingerprinter VARCHAR(64)` query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN fingerprinter VARCHAR(64)`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add ban_ip column to DBPREFIXfile_ban // add ban_ip column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip BOOL NOT NULL` query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip BOOL NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }
// add ban_ip_message column to DBPREFIXfile_ban // add ban_ip_message column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig) dataType, err = common.ColumnType(ctx, db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig)
if err != nil { if err != nil {
return err return err
} }
if dataType == "" { if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip_message TEXT` query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip_message TEXT`
if _, err = db.ExecTxSQL(tx, query); err != nil { if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err return err
} }
} }

87
cmd/gochan-migration/main.go
View file

@ -14,81 +14,61 @@ import (
"github.com/gochan-org/gochan/pkg/gcsql" "github.com/gochan-org/gochan/pkg/gcsql"
) )
const (
banner = `Welcome to the gochan database migration tool for gochan %s!
Make sure you check the README and/or the -h command line flag, and back up your current database before you use it.
`
migrateCompleteTxt = `Database migration successful!
To migrate the uploads for each board, move or copy the uploads to /path/to/gochan/document/root/<boardname>/src/
Then copy the thumbnails to /path/to/gochan/documentroot/<boardname>/thumb/
Then start the gochan server and go to http://yoursite/manage/rebuildall to generate the html files
for the threads and board pages`
allowedDirActions = "Valid values are noaction, copy, and move (defaults to noaction if unset)"
)
var ( var (
versionStr string versionStr string
migrator common.DBMigrator migrator common.DBMigrator
dbVersionStr string dbVersionStr string
) )
func cleanup() int { func cleanup() {
returnVal := 0
var err error var err error
exitCode := 0
if migrator != nil { if migrator != nil {
if err = migrator.Close(); err != nil { if err = migrator.Close(); err != nil {
returnVal = 1 common.LogError().Err(err).Caller().Msg("Error closing migrator")
log.Println("Error closing migrator:", err.Error()) exitCode = 1
} }
} }
if err = gcsql.Close(); err != nil { if err = gcsql.Close(); err != nil {
returnVal = 1 common.LogError().Err(err).Caller().Msg("Error closing SQL connection")
log.Println("Error closing SQL connection:", err.Error()) exitCode = 1
} }
return returnVal os.Exit(exitCode)
} }
func main() { func main() {
var options common.MigrationOptions var options common.MigrationOptions
var dirAction string
var updateDB bool var updateDB bool
log.SetFlags(0)
flag.BoolVar(&updateDB, "updatedb", false, "If this is set, gochan-migrate will check, and if needed, update gochan's database schema") flag.BoolVar(&updateDB, "updatedb", false, "If this is set, gochan-migrate will check, and if needed, update gochan's database schema")
flag.StringVar(&options.ChanType, "oldchan", "", "The imageboard we are migrating from (currently only pre2021 is supported, but more are coming") flag.StringVar(&options.ChanType, "oldchan", "", "The imageboard we are migrating from (currently only pre2021 is supported, but more are coming")
flag.StringVar(&options.OldChanConfig, "oldconfig", "", "The path to the old chan's configuration file") flag.StringVar(&options.OldChanConfig, "oldconfig", "", "The path to the old chan's configuration file")
// flag.StringVar(&dirAction, "diraction", "", "Action taken on each board directory after it has been migrated. "+allowedDirActions)
flag.Parse() flag.Parse()
config.InitConfig(versionStr) config.InitConfig(versionStr)
err := common.InitMigrationLog()
if err != nil {
log.Fatalln("Unable to initialize migration log:", err.Error())
}
fatalEv := common.LogFatal()
defer func() {
cleanup()
fatalEv.Discard()
}()
if !updateDB && (options.ChanType == "" || options.OldChanConfig == "") { if !updateDB && (options.ChanType == "" || options.OldChanConfig == "") {
flag.PrintDefaults() flag.PrintDefaults()
log.Fatal("Missing required oldchan value") fatalEv.Msg("Missing required oldchan value")
return
} else if updateDB { } else if updateDB {
options.ChanType = "gcupdate" options.ChanType = "gcupdate"
} }
switch dirAction { fatalEv.Str("chanType", options.ChanType)
case "":
fallthrough
case "noaction":
options.DirAction = common.DirNoAction
case "copy":
options.DirAction = common.DirCopy
case "move":
options.DirAction = common.DirMove
default:
log.Fatalln("Invalid diraction value. " + allowedDirActions)
}
log.Printf(banner, versionStr)
switch options.ChanType { switch options.ChanType {
case "gcupdate": case "gcupdate":
targetDBVer, err := strconv.Atoi(dbVersionStr) targetDBVer, err := strconv.Atoi(dbVersionStr)
if err != nil { if err != nil {
log.Fatalf("Invalid database version string %q, unable to parse to int", dbVersionStr) fatalEv.Err(err).Caller().Msg("Invalid database version string, unable to parse as integer")
} }
migrator = &gcupdate.GCDatabaseUpdater{ migrator = &gcupdate.GCDatabaseUpdater{
TargetDBVer: targetDBVer, TargetDBVer: targetDBVer,
@ -100,44 +80,31 @@ func main() {
case "tinyboard": case "tinyboard":
fallthrough fallthrough
default: default:
log.Fatalf( fatalEv.Msg("Unsupported chan type, Currently only pre2021 database migration is supported")
"Unsupported chan type %q, Currently only pre2021 database migration is supported\n",
options.ChanType)
return
} }
config.InitConfig(versionStr) config.InitConfig(versionStr)
var err error
if !updateDB { if !updateDB {
sqlCfg := config.GetSQLConfig() sqlCfg := config.GetSQLConfig()
err = gcsql.ConnectToDB(&sqlCfg) err = gcsql.ConnectToDB(&sqlCfg)
if err != nil { if err != nil {
log.Fatalf("Failed to connect to the database: %s", err.Error()) fatalEv.Err(err).Caller().Msg("Failed to connect to the database")
} }
if err = gcsql.CheckAndInitializeDatabase(sqlCfg.DBtype, dbVersionStr); err != nil { if err = gcsql.CheckAndInitializeDatabase(sqlCfg.DBtype, dbVersionStr); err != nil {
log.Fatalf("Failed to initialize the database: %s", err.Error()) fatalEv.Err(err).Caller().Msg("Unable to initialize the database")
} }
} }
if err = migrator.Init(&options); err != nil { if err = migrator.Init(&options); err != nil {
cleanup() fatalEv.Err(err).Caller().Msg("Unable to initialize migrator")
log.Fatalf("Unable to initialize %s migrator: %s\n",
options.ChanType, err.Error())
return
} }
var migrated bool
var migrated bool
if migrated, err = migrator.MigrateDB(); err != nil { if migrated, err = migrator.MigrateDB(); err != nil {
cleanup() fatalEv.Msg("Unable to migrate database")
log.Fatalln("Error migrating database:", err.Error())
} }
if migrated { if migrated {
log.Println("Database is already migrated") common.LogInfo().Msg("Database is already migrated")
return
}
if updateDB {
log.Println("Database schema updated successfully")
} else { } else {
log.Println(migrateCompleteTxt) common.LogInfo().Str("chanType", options.ChanType).Msg("Database migration complete")
} }
os.Exit(cleanup())
} }

4
pkg/gcsql/provisioning.go
View file

@ -25,7 +25,7 @@ var (
ErrCorruptedDB = errors.New("database contains gochan prefixed tables but is missing versioning tables (possibly corrupted)") ErrCorruptedDB = errors.New("database contains gochan prefixed tables but is missing versioning tables (possibly corrupted)")
ErrDeprecatedDB = errors.New("database layout is deprecated, please run gochan-migration -updatedb") ErrDeprecatedDB = errors.New("database layout is deprecated, please run gochan-migration -updatedb")
ErrInvalidDBVersion = errors.New("invalid version flag returned by GetCompleteDatabaseVersion()") ErrInvalidDBVersion = errors.New("invalid version flag returned by GetCompleteDatabaseVersion()")
targetDatabaseVersion = 3 targetDatabaseVersion = -1
) )
func initDB(initFile string) error { func initDB(initFile string) error {
@ -84,10 +84,12 @@ func GetCompleteDatabaseVersion() (dbVersion, dbFlag int, err error) {
// CheckAndInitializeDatabase checks the validity of the database and initialises it if it is empty // CheckAndInitializeDatabase checks the validity of the database and initialises it if it is empty
func CheckAndInitializeDatabase(dbType string, targetDbVersionStr string) (err error) { func CheckAndInitializeDatabase(dbType string, targetDbVersionStr string) (err error) {
if targetDatabaseVersion == -1 {
targetDatabaseVersion, err = strconv.Atoi(targetDbVersionStr) targetDatabaseVersion, err = strconv.Atoi(targetDbVersionStr)
if err != nil { if err != nil {
return err return err
} }
}
dbVersion, versionFlag, err := GetCompleteDatabaseVersion() dbVersion, versionFlag, err := GetCompleteDatabaseVersion()
if err != nil { if err != nil {

6
pkg/gcutil/logger.go
View file

@ -65,10 +65,10 @@ func LogDiscard(events ...*zerolog.Event) {
} }
} }
// isTerminal returns true if the ModeCharDevice bit is set, meaning that // RunningInTerminal returns true if the ModeCharDevice bit is set, meaning that
// gochan is probably running in a standard terminal and not being piped // gochan is probably running in a standard terminal and not being piped
// to a file // to a file
func isTerminal() bool { func RunningInTerminal() bool {
fi, err := os.Stdout.Stat() fi, err := os.Stdout.Stat()
if err != nil { if err != nil {
return false return false
@ -91,7 +91,7 @@ func initLog(logPath string, logToConsole bool) (err error) {
var writer io.Writer var writer io.Writer
if logToConsole { if logToConsole {
cw := zerolog.NewConsoleWriter() cw := zerolog.NewConsoleWriter()
cw.NoColor = !isTerminal() cw.NoColor = !RunningInTerminal()
writer = zerolog.MultiLevelWriter(logFile, cw) writer = zerolog.MultiLevelWriter(logFile, cw)
} else { } else {
writer = logFile writer = logFile