gochan/gochan
1
0
Fork 0
mirror of https://github.com/Eggbertx/gochan.git synced 2025-08-02 10:56:25 -07:00
Code Releases Activity

Use more context/timeout queries in gochan-migrate, start using zerolog

Browse source
This commit is contained in:
Eggbertx 2024-10-13 13:43:08 -07:00
parent 9d003d89a3
commit eb06047055
11 changed files with 307 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

50
cmd/gochan-migration/internal/common/filters.go
View file

@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"strings"
"time"
@ -39,7 +40,15 @@ func AddFilterTables(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig
// MigrateFileBans migrates file checksum and image fingerprint bans to the filter table
func MigrateFileBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,checksum,fingerprinter,ban_ip,ban_ip_message FROM DBPREFIXfile_ban`)
fileBanTableExists, err := TableExists(ctx, db, tx, "DBPREFIXfilename_ban", cfg)
if err != nil {
return err
}
if !fileBanTableExists {
// no filename bans to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_id,staff_id,staff_note,issued_at,checksum,fingerprinter,ban_ip,ban_ip_message FROM DBPREFIXfile_ban`)
if err != nil {
return err
}
@ -106,10 +115,19 @@ func MigrateFileBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *confi
// MigrateFilenameBans migrates filename bans to the filter table
func MigrateFilenameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,filename,is_regex FROM DBPREFIXfilename_ban`)
filenameBanTableExists, err := TableExists(ctx, db, tx, "DBPREFIXfilename_ban", cfg)
if err != nil {
return err
}
if !filenameBanTableExists {
// no filename bans to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_id,staff_id,staff_note,issued_at,filename,is_regex FROM DBPREFIXfilename_ban`)
if err != nil {
fmt.Println("query error")
return err
}
defer rows.Close()
var fnBanBoardID *int
@ -162,10 +180,19 @@ func MigrateFilenameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *c
// MigrateUsernameBans migrates poster name bans to the filter table
func MigrateUsernameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,username,is_regex FROM DBPREFIXusername_ban`)
usernameBanTableExists, err := TableExists(ctx, db, tx, "DBPREFIXusername_ban", cfg)
if err != nil {
return err
}
if !usernameBanTableExists {
// no name bans to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_id,staff_id,staff_note,issued_at,username,is_regex FROM DBPREFIXusername_ban`)
if err != nil {
fmt.Println("MigrateUsernameBans rows error")
return err
}
defer rows.Close()
var unBanBoardID *int
@ -184,7 +211,7 @@ func MigrateUsernameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *c
}
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilters(staff_id, staff_note, issued_at, match_action, match_detail, is_active) VALUES(?,?,?,?,?)`,
`INSERT INTO DBPREFIXfilters(staff_id, staff_note, issued_at, match_action, match_detail, is_active) VALUES(?,?,?,?,?,?)`,
unBanStaffID, unBanStaffNote, unBanIssuedAt, "reject", "", true,
); err != nil {
return err
@ -213,16 +240,24 @@ func MigrateUsernameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *c
return err
}
}
return rows.Close()
}
// MigrateWordfilters migrates pre-filter wordfilters to the filter table
func MigrateWordfilters(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_dirs, staff_id, staff_note, issued_at, search, is_regex, change_to FROM DBPREFIXwordfilters`)
wordfiltersTableExists, err := TableExists(ctx, db, tx, "DBPREFIXwordfilters", sqlConfig)
if err != nil {
return err
}
if !wordfiltersTableExists {
// no wordfilters to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_dirs, staff_id, staff_note, issued_at, search, is_regex, change_to FROM DBPREFIXwordfilters`)
if err != nil {
fmt.Println("MigrateWordfilters rows error")
return err
}
defer rows.Close()
var boardDirsPtr *string
@ -281,7 +316,8 @@ func MigrateWordfilters(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConf
}
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilter_conditions(filter_id, match_mode, search, field) VALUES(?,?,?,'body')`, filterID, matchMode, search,
`INSERT INTO DBPREFIXfilter_conditions(filter_id, match_mode, search, field) VALUES(?,?,?,'body')`,
filterID, matchMode, search,
); err != nil {
return err
}

3
cmd/gochan-migration/internal/common/handler.go
View file

@ -57,7 +57,8 @@ type DBMigrator interface {
// will exit
IsMigrated() (bool, error)
// MigrateDB migrates the imageboard data (posts, boards, etc) to the new database
// MigrateDB migrates the imageboard data (posts, boards, etc) to the new database. It is
// assumed that MigrateDB will handle logging any errors that occur during the migration
MigrateDB() (bool, error)
// MigrateBoards gets info about the old boards in the board table and inserts each one

67
cmd/gochan-migration/internal/common/logger.go Normal file
View file

@ -0,0 +1,67 @@
package common
import (
"io"
"io/fs"
"os"
"path"
"github.com/gochan-org/gochan/pkg/config"
"github.com/gochan-org/gochan/pkg/gcutil"
"github.com/rs/zerolog"
)
const (
logFlags = os.O_CREATE | os.O_APPEND | os.O_WRONLY
logFileMode fs.FileMode = 0644
)
var (
migrationLogFile *os.File
migrationLog zerolog.Logger
)
func InitMigrationLog() (err error) {
if migrationLogFile != nil {
// Migration log already initialized
return nil
}
logPath := path.Join(config.GetSystemCriticalConfig().LogDir, "migration.log")
migrationLogFile, err = os.OpenFile(logPath, logFlags, logFileMode)
if err != nil {
return err
}
var writer io.Writer
cw := zerolog.NewConsoleWriter()
cw.NoColor = !gcutil.RunningInTerminal()
writer = zerolog.MultiLevelWriter(migrationLogFile, cw)
migrationLog = zerolog.New(writer).With().Timestamp().Logger()
return nil
}
func Logger() *zerolog.Logger {
return &migrationLog
}
func LogInfo() *zerolog.Event {
return migrationLog.Info()
}
func LogWarning() *zerolog.Event {
return migrationLog.Warn()
}
func LogError() *zerolog.Event {
return migrationLog.Error()
}
func LogFatal() *zerolog.Event {
return migrationLog.Fatal()
}
func CloseLog() error {
if migrationLogFile == nil {
return nil
}
return migrationLogFile.Close()
}

22
cmd/gochan-migration/internal/common/sqlutil.go
View file

@ -1,6 +1,7 @@
package common
import (
"context"
"database/sql"
"errors"
"fmt"
@ -20,7 +21,7 @@ var (
// ColumnType returns a string representation of the column's data type. It does not return an error
// if the column does not exist, instead returning an empty string.
func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string, sqlConfig *config.SQLConfig) (string, error) {
func ColumnType(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string, sqlConfig *config.SQLConfig) (string, error) {
var query string
var dataType string
var err error
@ -32,7 +33,7 @@ func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string,
query = `SELECT DATA_TYPE FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1`
params = []any{dbName, tableName, columnName}
case "postgresql":
case "postgres", "postgresql":
query = `SELECT data_type FROM information_schema.columns
WHERE (table_schema = ? OR table_schema = 'public')
AND table_name = ? AND column_name = ? LIMIT 1`
@ -43,7 +44,7 @@ func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string,
default:
return "", gcsql.ErrUnsupportedDB
}
err = db.QueryRowTxSQL(tx, query, params, []any{&dataType})
err = db.QueryRowContextSQL(ctx, tx, query, params, []any{&dataType})
if errors.Is(err, sql.ErrNoRows) {
return "", nil
}
@ -51,26 +52,21 @@ func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string,
}
// TableExists returns true if the given table exists in the given database, and an error if one occured
func TableExists(db *gcsql.GCDB, tx *sql.Tx, tableName string, sqlConfig *config.SQLConfig) (bool, error) {
func TableExists(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, tableName string, sqlConfig *config.SQLConfig) (bool, error) {
tableName = strings.ReplaceAll(tableName, "DBPREFIX", sqlConfig.DBprefix)
dbName := sqlConfig.DBname
var query string
var params []any
switch sqlConfig.DBtype {
case "mysql":
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?`
params = []any{dbName, tableName}
case "postgresql":
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE table_catalog = ? AND table_name = ?`
params = []any{dbName, tableName}
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?`
case "postgres", "postgresql":
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE table_catalog = CURRENT_DATABASE() AND table_name = ?`
case "sqlite3":
query = `SELECT COUNT(*) FROM sqlite_master WHERE name = ? AND type = 'table'`
params = []any{tableName}
default:
return false, gcsql.ErrUnsupportedDB
}
var count int
err := db.QueryRowTxSQL(tx, query, params, []any{&count})
err := db.QueryRowContextSQL(ctx, tx, query, []any{tableName}, []any{&count})
return count == 1, err
}

36
cmd/gochan-migration/internal/gcupdate/gcupdate.go
View file

@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
"time"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
"github.com/gochan-org/gochan/pkg/config"
@ -51,7 +52,8 @@ func (dbu *GCDatabaseUpdater) MigrateDB() (bool, error) {
}
sqlConfig := config.GetSQLConfig()
ctx := context.Background()
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
tx, err := dbu.db.BeginTx(ctx, &sql.TxOptions{
Isolation: 0,
ReadOnly: false,
@ -63,46 +65,56 @@ func (dbu *GCDatabaseUpdater) MigrateDB() (bool, error) {
switch sqlConfig.DBtype {
case "mysql":
err = updateMysqlDB(dbu.db, tx, &sqlConfig)
err = updateMysqlDB(ctx, dbu.db, tx, &sqlConfig)
case "postgres":
err = updatePostgresDB(dbu.db, tx, &sqlConfig)
err = updatePostgresDB(ctx, dbu.db, tx, &sqlConfig)
case "sqlite3":
err = updateSqliteDB(dbu.db, tx, &sqlConfig)
err = updateSqliteDB(ctx, dbu.db, tx, &sqlConfig)
}
if err != nil {
return false, err
}
filterTableExists, err := common.TableExists(dbu.db, nil, "DBPREFIXfilters", &sqlConfig)
// commit the transaction and start a new one (to avoid deadlocks)
if err = tx.Commit(); err != nil {
return false, err
}
if err = ctx.Err(); err != nil {
return false, err
}
filterTableExists, err := common.TableExists(ctx, dbu.db, nil, "DBPREFIXfilters", &sqlConfig)
if err != nil {
return false, err
}
if !filterTableExists {
// DBPREFIXfilters not found, create it and migrate data from DBPREFIXfile_bans, DBPREFIXfilename_bans, and DBPREFIXusername_bans,
if err = common.AddFilterTables(ctx, dbu.db, tx, &sqlConfig); err != nil {
if err = common.AddFilterTables(ctx, dbu.db, nil, &sqlConfig); err != nil {
return false, err
}
if err = common.MigrateFileBans(ctx, dbu.db, tx, &sqlConfig); err != nil {
if err = common.MigrateFileBans(ctx, dbu.db, nil, &sqlConfig); err != nil {
return false, err
}
if err = common.MigrateFilenameBans(ctx, dbu.db, tx, &sqlConfig); err != nil {
if err = common.MigrateFilenameBans(ctx, dbu.db, nil, &sqlConfig); err != nil {
return false, err
}
if err = common.MigrateUsernameBans(ctx, dbu.db, tx, &sqlConfig); err != nil {
if err = common.MigrateUsernameBans(ctx, dbu.db, nil, &sqlConfig); err != nil {
return false, err
}
if err = common.MigrateWordfilters(ctx, dbu.db, tx, &sqlConfig); err != nil {
if err = common.MigrateWordfilters(ctx, dbu.db, nil, &sqlConfig); err != nil {
return false, err
}
}
query := `UPDATE DBPREFIXdatabase_version SET version = ? WHERE component = 'gochan'`
_, err = dbu.db.ExecTxSQL(tx, query, dbu.TargetDBVer)
_, err = dbu.db.ExecContextSQL(ctx, nil, query, dbu.TargetDBVer)
if err != nil {
return false, err
}
return false, tx.Commit()
// return false, tx.Commit()
return false, nil
}
func (*GCDatabaseUpdater) MigrateBoards() error {

149
cmd/gochan-migration/internal/gcupdate/updatemysql.go
View file

@ -1,6 +1,7 @@
package gcupdate
import (
"context"
"database/sql"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
@ -9,41 +10,52 @@ import (
"github.com/gochan-org/gochan/pkg/gcutil"
)
func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
var numConstraints int
var err error
func updateMysqlDB(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
var query string
var dataType string
dbName := sqlConfig.DBname
query := `SELECT COUNT(*) FROM information_schema.TABLE_CONSTRAINTS
WHERE CONSTRAINT_NAME = 'wordfilters_board_id_fk'
AND TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'DBPREFIXwordfilters'`
if err = db.QueryRowTxSQL(tx, query, nil, []any{&numConstraints}); err != nil {
return err
}
if numConstraints > 0 {
query = `ALTER TABLE DBPREFIXwordfilters DROP FOREIGN KEY wordfilters_board_id_fk`
} else {
query = ""
}
dataType, err := common.ColumnType(db, tx, "board_dirs", "DBPREFIXwordfilters", sqlConfig)
wordfiltersTableExists, err := common.TableExists(ctx, db, tx, "DBPREFIXwordfilters", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if wordfiltersTableExists {
// wordfilters table is going to be migrated by the end of the update, but we want to make sure its legacy data is migrated first
// so it can be properly merged into the filter table
var numConstraints int
query = `SELECT COUNT(*) FROM information_schema.TABLE_CONSTRAINTS
WHERE CONSTRAINT_NAME = 'wordfilters_board_id_fk'
AND TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'DBPREFIXwordfilters'`
if err = db.QueryRowContextSQL(ctx, tx, query, nil, []any{&numConstraints}); err != nil {
return err
}
if numConstraints > 0 {
query = `ALTER TABLE DBPREFIXwordfilters DROP FOREIGN KEY wordfilters_board_id_fk`
} else {
query = ""
}
dataType, err = common.ColumnType(ctx, db, tx, "board_dirs", "DBPREFIXwordfilters", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXwordfilters ADD COLUMN board_dirs varchar(255) DEFAULT '*'`
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
}
// Yay, collation! Everybody loves MySQL's default collation!
query = `ALTER DATABASE ` + dbName + ` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci`
if _, err = tx.Exec(query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
query = `SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?`
rows, err := db.QuerySQL(query, dbName)
rows, err := db.QueryContextSQL(ctx, tx, query, dbName)
if err != nil {
return err
}
@ -57,14 +69,15 @@ func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) erro
return err
}
query = `ALTER TABLE ` + tableName + ` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`
if _, err = tx.Exec(query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
if err = rows.Close(); err != nil {
return err
}
dataType, err = common.ColumnType(db, tx, "ip", "DBPREFIXip_ban", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "ip", "DBPREFIXip_ban", sqlConfig)
if err != nil {
return err
}
@ -73,11 +86,11 @@ func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) erro
query = `ALTER TABLE DBPREFIXip_ban
ADD COLUMN IF NOT EXISTS range_start VARBINARY(16) NOT NULL,
ADD COLUMN IF NOT EXISTS range_end VARBINARY(16) NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
// convert ban IP string to IP range
if rows, err = db.QuerySQL(`SELECT id, ip FROM DBPREFIXip_ban`); err != nil {
if rows, err = db.QueryContextSQL(ctx, tx, `SELECT id, ip FROM DBPREFIXip_ban`); err != nil {
return err
}
var rangeStart, rangeEnd string
@ -92,11 +105,11 @@ func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) erro
}
query = `UPDATE DBPREFIXip_ban
SET range_start = INET6_ATON(?), range_end = INET6_ATON(?) WHERE id = ?`
if _, err = db.ExecTxSQL(tx, query, rangeStart, rangeEnd, id); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query, rangeStart, rangeEnd, id); err != nil {
return err
}
query = `ALTER TABLE DBPREFIXip_ban DROP COLUMN IF EXISTS ip`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
@ -106,123 +119,131 @@ func updateMysqlDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) erro
}
// Convert DBPREFIXposts.ip to from varchar to varbinary
dataType, err = common.ColumnType(db, tx, "ip", "DBPREFIXposts", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "ip", "DBPREFIXposts", sqlConfig)
if err != nil {
return err
}
if common.IsStringType(dataType) {
// rename `ip` to a temporary column to then be removed
query = `ALTER TABLE DBPREFIXposts CHANGE ip ip_str varchar(45)`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
query = `ALTER TABLE DBPREFIXposts
ADD COLUMN IF NOT EXISTS ip VARBINARY(16) NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
// convert post IP VARCHAR(45) to VARBINARY(16)
query = `UPDATE DBPREFIXposts SET ip = INET6_ATON(ip_str)`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
query = `ALTER TABLE DBPREFIXposts DROP COLUMN IF EXISTS ip_str`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// Convert DBPREFIXreports.ip to from varchar to varbinary
dataType, err = common.ColumnType(db, tx, "ip", "DBPREFIXreports", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "ip", "DBPREFIXreports", sqlConfig)
if err != nil {
return err
}
if common.IsStringType(dataType) {
// rename `ip` to a temporary column to then be removed
query = `ALTER TABLE DBPREFIXreports CHANGE ip ip_str varchar(45)`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
query = `ALTER TABLE DBPREFIXreports
ADD COLUMN IF NOT EXISTS ip VARBINARY(16) NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
// convert report IP VARCHAR(45) to VARBINARY(16)
query = `UPDATE DBPREFIXreports SET ip = INET6_ATON(ip_str)`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
query = `ALTER TABLE DBPREFIXreports DROP COLUMN IF EXISTS ip_str`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add flag column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "flag", "DBPREFIXposts", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "flag", "DBPREFIXposts", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add country column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "country", "DBPREFIXposts", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "country", "DBPREFIXposts", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add fingerprinter column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig)
fileBanTableExists, err := common.TableExists(ctx, db, tx, "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN fingerprinter VARCHAR(64)`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if fileBanTableExists {
// file ban table is going to be migrated by the end of the update, but we want to make sure its legacy data is migrated first
// so it can be properly merged into the filter table
// add fingerprinter column to DBPREFIXfile_ban
dataType, err = common.ColumnType(ctx, db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN fingerprinter VARCHAR(64)`
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add ban_ip column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip BOOL NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil {
// add ban_ip column to DBPREFIXfile_ban
dataType, err = common.ColumnType(ctx, db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
}
// add ban_ip_message column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip_message TEXT`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip BOOL NOT NULL`
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add ban_ip_message column to DBPREFIXfile_ban
dataType, err = common.ColumnType(ctx, db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip_message TEXT`
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
}
return nil

44
cmd/gochan-migration/internal/gcupdate/updatepostgres.go
View file

@ -1,6 +1,7 @@
package gcupdate
import (
"context"
"database/sql"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
@ -8,27 +9,28 @@ import (
"github.com/gochan-org/gochan/pkg/gcsql"
)
func updatePostgresDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
func updatePostgresDB(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
query := `ALTER TABLE DBPREFIXwordfilters
DROP CONSTRAINT IF EXISTS board_id_fk`
_, err := db.ExecSQL(query)
_, err := db.ExecContextSQL(ctx, tx, query)
if err != nil {
return err
}
query = `ALTER TABLE DBPREFIXwordfilters
ADD COLUMN IF NOT EXISTS board_dirs varchar(255) DEFAULT '*'`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
dataType, err := common.ColumnType(db, tx, "ip", "DBPREFIXposts", sqlConfig)
dataType, err := common.ColumnType(ctx, db, tx, "ip", "DBPREFIXposts", sqlConfig)
if err != nil {
return err
}
if common.IsStringType(dataType) {
// change ip column to temporary ip_str
query = `ALTER TABLE DBPREFIXposts RENAME COLUMN ip TO ip_str,`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
@ -36,22 +38,22 @@ func updatePostgresDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) e
// because it is non-nil
query = `ALTER TABLE DBPREFIXposts
ADD COLUMN IF NOT EXISTS ip INET NOT NULL DEFAULT '127.0.0.1'`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
query = `UPDATE TABLE DBPREFIXposts SET ip = ip_str`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
query = `ALTER TABLE DBPREFIXposts DROP COLUMN ip_str`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
dataType, err = common.ColumnType(db, tx, "ip", "DBPREFIXip_ban", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "ip", "DBPREFIXip_ban", sqlConfig)
if err != nil {
return err
}
@ -59,72 +61,72 @@ func updatePostgresDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) e
query = `ALTER TABLE DBPREFIXip_ban
ADD COLUMN IF NOT EXISTS range_start INET NOT NULL,
ADD COLUMN IF NOT EXISTS range_end INET NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
query = `UPDATE DBPREFIXip_ban SET range_start = ip::INET, SET range_end = ip::INET`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add flag column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "flag", "DBPREFIXposts", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "flag", "DBPREFIXposts", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add country column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "country", "DBPREFIXposts", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "country", "DBPREFIXposts", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add fingerprinter column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN fingerprinter VARCHAR(64)`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add ban_ip column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip BOOL NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add ban_ip_message column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip_message TEXT`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}

33
cmd/gochan-migration/internal/gcupdate/updatesqlite3.go
View file

@ -1,6 +1,7 @@
package gcupdate
import (
"context"
"database/sql"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
@ -8,13 +9,15 @@ import (
"github.com/gochan-org/gochan/pkg/gcsql"
)
func updateSqliteDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
func updateSqliteDB(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
var query string
_, err := db.ExecSQL(`PRAGMA foreign_keys = ON`)
_, err := db.ExecContextSQL(ctx, tx, `PRAGMA foreign_keys = ON`)
if err != nil {
return err
}
dataType, err := common.ColumnType(db, tx, "DBPREFIXwordfilters", "board_dirs", sqlConfig)
dataType, err := common.ColumnType(ctx, db, tx, "DBPREFIXwordfilters", "board_dirs", sqlConfig)
if err != nil {
return err
}
@ -26,7 +29,7 @@ func updateSqliteDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) err
}
// Add range_start column to DBPREFIXIp_ban if it doesn't exist
dataType, err = common.ColumnType(db, tx, "DBPREFIXip_ban", "range_start", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "DBPREFIXip_ban", "range_start", sqlConfig)
if err != nil {
return err
}
@ -38,7 +41,7 @@ func updateSqliteDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) err
}
// Add range_start column if it doesn't exist
dataType, err = common.ColumnType(db, tx, "DBPREFIXip_ban", "range_end", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "DBPREFIXip_ban", "range_end", sqlConfig)
if err != nil {
return err
}
@ -50,61 +53,61 @@ func updateSqliteDB(db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) err
}
// add flag column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "flag", "DBPREFIXposts", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "flag", "DBPREFIXposts", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add country column to DBPREFIXposts
dataType, err = common.ColumnType(db, tx, "country", "DBPREFIXposts", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "country", "DBPREFIXposts", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add fingerprinter column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "fingerprinter", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN fingerprinter VARCHAR(64)`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add ban_ip column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "ban_ip", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip BOOL NOT NULL`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}
// add ban_ip_message column to DBPREFIXfile_ban
dataType, err = common.ColumnType(db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig)
dataType, err = common.ColumnType(ctx, db, tx, "ban_ip_message", "DBPREFIXfile_ban", sqlConfig)
if err != nil {
return err
}
if dataType == "" {
query = `ALTER TABLE DBPREFIXfile_ban ADD COLUMN ban_ip_message TEXT`
if _, err = db.ExecTxSQL(tx, query); err != nil {
if _, err = db.ExecContextSQL(ctx, tx, query); err != nil {
return err
}
}

87
cmd/gochan-migration/main.go
View file

@ -14,81 +14,61 @@ import (
"github.com/gochan-org/gochan/pkg/gcsql"
)
const (
banner = `Welcome to the gochan database migration tool for gochan %s!
Make sure you check the README and/or the -h command line flag, and back up your current database before you use it.
`
migrateCompleteTxt = `Database migration successful!
To migrate the uploads for each board, move or copy the uploads to /path/to/gochan/document/root/<boardname>/src/
Then copy the thumbnails to /path/to/gochan/documentroot/<boardname>/thumb/
Then start the gochan server and go to http://yoursite/manage/rebuildall to generate the html files
for the threads and board pages`
allowedDirActions = "Valid values are noaction, copy, and move (defaults to noaction if unset)"
)
var (
versionStr string
migrator common.DBMigrator
dbVersionStr string
)
func cleanup() int {
returnVal := 0
func cleanup() {
var err error
exitCode := 0
if migrator != nil {
if err = migrator.Close(); err != nil {
returnVal = 1
log.Println("Error closing migrator:", err.Error())
common.LogError().Err(err).Caller().Msg("Error closing migrator")
exitCode = 1
}
}
if err = gcsql.Close(); err != nil {
returnVal = 1
log.Println("Error closing SQL connection:", err.Error())
common.LogError().Err(err).Caller().Msg("Error closing SQL connection")
exitCode = 1
}
return returnVal
os.Exit(exitCode)
}
func main() {
var options common.MigrationOptions
var dirAction string
var updateDB bool
log.SetFlags(0)
flag.BoolVar(&updateDB, "updatedb", false, "If this is set, gochan-migrate will check, and if needed, update gochan's database schema")
flag.StringVar(&options.ChanType, "oldchan", "", "The imageboard we are migrating from (currently only pre2021 is supported, but more are coming")
flag.StringVar(&options.OldChanConfig, "oldconfig", "", "The path to the old chan's configuration file")
// flag.StringVar(&dirAction, "diraction", "", "Action taken on each board directory after it has been migrated. "+allowedDirActions)
flag.Parse()
config.InitConfig(versionStr)
err := common.InitMigrationLog()
if err != nil {
log.Fatalln("Unable to initialize migration log:", err.Error())
}
fatalEv := common.LogFatal()
defer func() {
cleanup()
fatalEv.Discard()
}()
if !updateDB && (options.ChanType == "" || options.OldChanConfig == "") {
flag.PrintDefaults()
log.Fatal("Missing required oldchan value")
return
fatalEv.Msg("Missing required oldchan value")
} else if updateDB {
options.ChanType = "gcupdate"
}
switch dirAction {
case "":
fallthrough
case "noaction":
options.DirAction = common.DirNoAction
case "copy":
options.DirAction = common.DirCopy
case "move":
options.DirAction = common.DirMove
default:
log.Fatalln("Invalid diraction value. " + allowedDirActions)
}
fatalEv.Str("chanType", options.ChanType)
log.Printf(banner, versionStr)
switch options.ChanType {
case "gcupdate":
targetDBVer, err := strconv.Atoi(dbVersionStr)
if err != nil {
log.Fatalf("Invalid database version string %q, unable to parse to int", dbVersionStr)
fatalEv.Err(err).Caller().Msg("Invalid database version string, unable to parse as integer")
}
migrator = &gcupdate.GCDatabaseUpdater{
TargetDBVer: targetDBVer,
@ -100,44 +80,31 @@ func main() {
case "tinyboard":
fallthrough
default:
log.Fatalf(
"Unsupported chan type %q, Currently only pre2021 database migration is supported\n",
options.ChanType)
return
fatalEv.Msg("Unsupported chan type, Currently only pre2021 database migration is supported")
}
config.InitConfig(versionStr)
var err error
if !updateDB {
sqlCfg := config.GetSQLConfig()
err = gcsql.ConnectToDB(&sqlCfg)
if err != nil {
log.Fatalf("Failed to connect to the database: %s", err.Error())
fatalEv.Err(err).Caller().Msg("Failed to connect to the database")
}
if err = gcsql.CheckAndInitializeDatabase(sqlCfg.DBtype, dbVersionStr); err != nil {
log.Fatalf("Failed to initialize the database: %s", err.Error())
fatalEv.Err(err).Caller().Msg("Unable to initialize the database")
}
}
if err = migrator.Init(&options); err != nil {
cleanup()
log.Fatalf("Unable to initialize %s migrator: %s\n",
options.ChanType, err.Error())
return
fatalEv.Err(err).Caller().Msg("Unable to initialize migrator")
}
var migrated bool
var migrated bool
if migrated, err = migrator.MigrateDB(); err != nil {
cleanup()
log.Fatalln("Error migrating database:", err.Error())
fatalEv.Msg("Unable to migrate database")
}
if migrated {
log.Println("Database is already migrated")
return
}
if updateDB {
log.Println("Database schema updated successfully")
common.LogInfo().Msg("Database is already migrated")
} else {
log.Println(migrateCompleteTxt)
common.LogInfo().Str("chanType", options.ChanType).Msg("Database migration complete")
}
os.Exit(cleanup())
}

10
pkg/gcsql/provisioning.go
View file

@ -25,7 +25,7 @@ var (
ErrCorruptedDB = errors.New("database contains gochan prefixed tables but is missing versioning tables (possibly corrupted)")
ErrDeprecatedDB = errors.New("database layout is deprecated, please run gochan-migration -updatedb")
ErrInvalidDBVersion = errors.New("invalid version flag returned by GetCompleteDatabaseVersion()")
targetDatabaseVersion = 3
targetDatabaseVersion = -1
)
func initDB(initFile string) error {
@ -84,9 +84,11 @@ func GetCompleteDatabaseVersion() (dbVersion, dbFlag int, err error) {
// CheckAndInitializeDatabase checks the validity of the database and initialises it if it is empty
func CheckAndInitializeDatabase(dbType string, targetDbVersionStr string) (err error) {
targetDatabaseVersion, err = strconv.Atoi(targetDbVersionStr)
if err != nil {
return err
if targetDatabaseVersion == -1 {
targetDatabaseVersion, err = strconv.Atoi(targetDbVersionStr)
if err != nil {
return err
}
}
dbVersion, versionFlag, err := GetCompleteDatabaseVersion()

6
pkg/gcutil/logger.go
View file

@ -65,10 +65,10 @@ func LogDiscard(events ...*zerolog.Event) {
}
}
// isTerminal returns true if the ModeCharDevice bit is set, meaning that
// RunningInTerminal returns true if the ModeCharDevice bit is set, meaning that
// gochan is probably running in a standard terminal and not being piped
// to a file
func isTerminal() bool {
func RunningInTerminal() bool {
fi, err := os.Stdout.Stat()
if err != nil {
return false
@ -91,7 +91,7 @@ func initLog(logPath string, logToConsole bool) (err error) {
var writer io.Writer
if logToConsole {
cw := zerolog.NewConsoleWriter()
cw.NoColor = !isTerminal()
cw.NoColor = !RunningInTerminal()
writer = zerolog.MultiLevelWriter(logFile, cw)
} else {
writer = logFile