gochan/gochan
1
0
Fork 0
mirror of https://github.com/Eggbertx/gochan.git synced 2025-09-10 02:26:23 -07:00
Code Releases Activity

Use more context/timeout queries in gochan-migrate, start using zerolog

Browse source
This commit is contained in:
Eggbertx 2024-10-13 13:43:08 -07:00
parent 9d003d89a3
commit eb06047055
11 changed files with 307 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

50
cmd/gochan-migration/internal/common/filters.go
View file

@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"strings"
"time"
@ -39,7 +40,15 @@ func AddFilterTables(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig
// MigrateFileBans migrates file checksum and image fingerprint bans to the filter table
func MigrateFileBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,checksum,fingerprinter,ban_ip,ban_ip_message FROM DBPREFIXfile_ban`)
fileBanTableExists, err := TableExists(ctx, db, tx, "DBPREFIXfilename_ban", cfg)
if err != nil {
return err
}
if !fileBanTableExists {
// no filename bans to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_id,staff_id,staff_note,issued_at,checksum,fingerprinter,ban_ip,ban_ip_message FROM DBPREFIXfile_ban`)
if err != nil {
return err
}
@ -106,10 +115,19 @@ func MigrateFileBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *confi
// MigrateFilenameBans migrates filename bans to the filter table
func MigrateFilenameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,filename,is_regex FROM DBPREFIXfilename_ban`)
filenameBanTableExists, err := TableExists(ctx, db, tx, "DBPREFIXfilename_ban", cfg)
if err != nil {
return err
}
if !filenameBanTableExists {
// no filename bans to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_id,staff_id,staff_note,issued_at,filename,is_regex FROM DBPREFIXfilename_ban`)
if err != nil {
fmt.Println("query error")
return err
}
defer rows.Close()
var fnBanBoardID *int
@ -162,10 +180,19 @@ func MigrateFilenameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *c
// MigrateUsernameBans migrates poster name bans to the filter table
func MigrateUsernameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,username,is_regex FROM DBPREFIXusername_ban`)
usernameBanTableExists, err := TableExists(ctx, db, tx, "DBPREFIXusername_ban", cfg)
if err != nil {
return err
}
if !usernameBanTableExists {
// no name bans to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_id,staff_id,staff_note,issued_at,username,is_regex FROM DBPREFIXusername_ban`)
if err != nil {
fmt.Println("MigrateUsernameBans rows error")
return err
}
defer rows.Close()
var unBanBoardID *int
@ -184,7 +211,7 @@ func MigrateUsernameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *c
}
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilters(staff_id, staff_note, issued_at, match_action, match_detail, is_active) VALUES(?,?,?,?,?)`,
`INSERT INTO DBPREFIXfilters(staff_id, staff_note, issued_at, match_action, match_detail, is_active) VALUES(?,?,?,?,?,?)`,
unBanStaffID, unBanStaffNote, unBanIssuedAt, "reject", "", true,
); err != nil {
return err
@ -213,16 +240,24 @@ func MigrateUsernameBans(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, cfg *c
return err
}
}
return rows.Close()
}
// MigrateWordfilters migrates pre-filter wordfilters to the filter table
func MigrateWordfilters(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_dirs, staff_id, staff_note, issued_at, search, is_regex, change_to FROM DBPREFIXwordfilters`)
wordfiltersTableExists, err := TableExists(ctx, db, tx, "DBPREFIXwordfilters", sqlConfig)
if err != nil {
return err
}
if !wordfiltersTableExists {
// no wordfilters to migrate (database partially migrated?)
return nil
}
rows, err := db.QueryContextSQL(ctx, tx, `SELECT board_dirs, staff_id, staff_note, issued_at, search, is_regex, change_to FROM DBPREFIXwordfilters`)
if err != nil {
fmt.Println("MigrateWordfilters rows error")
return err
}
defer rows.Close()
var boardDirsPtr *string
@ -281,7 +316,8 @@ func MigrateWordfilters(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, sqlConf
}
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilter_conditions(filter_id, match_mode, search, field) VALUES(?,?,?,'body')`, filterID, matchMode, search,
`INSERT INTO DBPREFIXfilter_conditions(filter_id, match_mode, search, field) VALUES(?,?,?,'body')`,
filterID, matchMode, search,
); err != nil {
return err
}

3
cmd/gochan-migration/internal/common/handler.go
View file

@ -57,7 +57,8 @@ type DBMigrator interface {
// will exit
IsMigrated() (bool, error)
// MigrateDB migrates the imageboard data (posts, boards, etc) to the new database
// MigrateDB migrates the imageboard data (posts, boards, etc) to the new database. It is
// assumed that MigrateDB will handle logging any errors that occur during the migration
MigrateDB() (bool, error)
// MigrateBoards gets info about the old boards in the board table and inserts each one

67
cmd/gochan-migration/internal/common/logger.go Normal file
View file

@ -0,0 +1,67 @@
package common
import (
"io"
"io/fs"
"os"
"path"
"github.com/gochan-org/gochan/pkg/config"
"github.com/gochan-org/gochan/pkg/gcutil"
"github.com/rs/zerolog"
)
const (
logFlags = os.O_CREATE | os.O_APPEND | os.O_WRONLY
logFileMode fs.FileMode = 0644
)
var (
migrationLogFile *os.File
migrationLog zerolog.Logger
)
func InitMigrationLog() (err error) {
if migrationLogFile != nil {
// Migration log already initialized
return nil
}
logPath := path.Join(config.GetSystemCriticalConfig().LogDir, "migration.log")
migrationLogFile, err = os.OpenFile(logPath, logFlags, logFileMode)
if err != nil {
return err
}
var writer io.Writer
cw := zerolog.NewConsoleWriter()
cw.NoColor = !gcutil.RunningInTerminal()
writer = zerolog.MultiLevelWriter(migrationLogFile, cw)
migrationLog = zerolog.New(writer).With().Timestamp().Logger()
return nil
}
func Logger() *zerolog.Logger {
return &migrationLog
}
func LogInfo() *zerolog.Event {
return migrationLog.Info()
}
func LogWarning() *zerolog.Event {
return migrationLog.Warn()
}
func LogError() *zerolog.Event {
return migrationLog.Error()
}
func LogFatal() *zerolog.Event {
return migrationLog.Fatal()
}
func CloseLog() error {
if migrationLogFile == nil {
return nil
}
return migrationLogFile.Close()
}

22
cmd/gochan-migration/internal/common/sqlutil.go
View file

@ -1,6 +1,7 @@
package common
import (
"context"
"database/sql"
"errors"
"fmt"
@ -20,7 +21,7 @@ var (
// ColumnType returns a string representation of the column's data type. It does not return an error
// if the column does not exist, instead returning an empty string.
func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string, sqlConfig *config.SQLConfig) (string, error) {
func ColumnType(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string, sqlConfig *config.SQLConfig) (string, error) {
var query string
var dataType string
var err error
@ -32,7 +33,7 @@ func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string,
query = `SELECT DATA_TYPE FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1`
params = []any{dbName, tableName, columnName}
case "postgresql":
case "postgres", "postgresql":
query = `SELECT data_type FROM information_schema.columns
WHERE (table_schema = ? OR table_schema = 'public')
AND table_name = ? AND column_name = ? LIMIT 1`
@ -43,7 +44,7 @@ func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string,
default:
return "", gcsql.ErrUnsupportedDB
}
err = db.QueryRowTxSQL(tx, query, params, []any{&dataType})
err = db.QueryRowContextSQL(ctx, tx, query, params, []any{&dataType})
if errors.Is(err, sql.ErrNoRows) {
return "", nil
}
@ -51,26 +52,21 @@ func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string,
}
// TableExists returns true if the given table exists in the given database, and an error if one occured
func TableExists(db *gcsql.GCDB, tx *sql.Tx, tableName string, sqlConfig *config.SQLConfig) (bool, error) {
func TableExists(ctx context.Context, db *gcsql.GCDB, tx *sql.Tx, tableName string, sqlConfig *config.SQLConfig) (bool, error) {
tableName = strings.ReplaceAll(tableName, "DBPREFIX", sqlConfig.DBprefix)
dbName := sqlConfig.DBname
var query string
var params []any
switch sqlConfig.DBtype {
case "mysql":
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?`
params = []any{dbName, tableName}
case "postgresql":
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE table_catalog = ? AND table_name = ?`
params = []any{dbName, tableName}
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?`
case "postgres", "postgresql":
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE table_catalog = CURRENT_DATABASE() AND table_name = ?`
case "sqlite3":
query = `SELECT COUNT(*) FROM sqlite_master WHERE name = ? AND type = 'table'`
params = []any{tableName}
default:
return false, gcsql.ErrUnsupportedDB
}
var count int
err := db.QueryRowTxSQL(tx, query, params, []any{&count})
err := db.QueryRowContextSQL(ctx, tx, query, []any{tableName}, []any{&count})
return count == 1, err
}