gochan/gochan
1
0
Fork 0
mirror of https://github.com/Eggbertx/gochan.git synced 2025-08-16 18:16:24 -07:00
Code Releases Activity

Add migration stuff for filter tables

Browse source
This commit is contained in:
Eggbertx 2024-08-10 20:01:33 -07:00
parent c7f15e3d25
commit e12b2fb982
19 changed files with 295 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

180
cmd/gochan-migration/internal/common/filters.go Normal file
View file

@ -0,0 +1,180 @@
package common
import (
"context"
"database/sql"
"os"
"strings"
"time"
"github.com/gochan-org/gochan/pkg/config"
"github.com/gochan-org/gochan/pkg/gcsql"
)
// Used for db version 4 upgrade to create the filter tables from the respective SQL init file
func AddFilterTables(db *gcsql.GCDB, ctx context.Context, tx *sql.Tx, sqlConfig *config.SQLConfig) error {
filePath, err := getInitFilePath("initdb_" + sqlConfig.DBtype + ".sql")
if err != nil {
return err
}
ba, err := os.ReadFile(filePath)
if err != nil {
return err
}
sqlStr := commentRemover.ReplaceAllString(string(ba), " ")
sqlArr := strings.Split(sqlStr, ";")
for _, stmtStr := range sqlArr {
stmtStr = strings.TrimSpace(stmtStr)
if !strings.HasPrefix(stmtStr, "CREATE TABLE DBPREFIXfilter") {
continue
}
if _, err = db.ExecContextSQL(ctx, tx, stmtStr); err != nil {
return err
}
}
return nil
}
func MigrateFileBans(db *gcsql.GCDB, ctx context.Context, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,checksum,fingerprinter,ban_ip,ban_ip_message FROM DBPREFIXfile_ban`)
if err != nil {
return err
}
var fBanBoardID *int
var fBanStaffID int
var fBanStaffNote string
var fBanIssuedAt time.Time
var fBanChecksum string
var fBanFingerprinter *string
var fBanBanIP bool
var fBanBanIPMessage *string
var matchAction string
var detail string
var filterID int
var field string
for rows.Next() {
if err = rows.Scan(
&fBanBoardID, &fBanStaffID, &fBanStaffNote, &fBanIssuedAt, &fBanChecksum, &fBanFingerprinter, &fBanBanIP, &fBanBanIPMessage,
); err != nil {
return err
}
if fBanBanIP {
matchAction = "ban"
} else {
matchAction = "reject"
}
if fBanBanIPMessage == nil {
detail = ""
} else {
detail = *fBanBanIPMessage
}
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilters(staff_id, staff_note, issued_at, match_action, match_detail) VALUES(?,?,?,?,?)`,
fBanStaffID, fBanStaffNote, fBanIssuedAt, matchAction, detail); err != nil {
return err
}
if err = db.QueryRowContextSQL(ctx, tx, `SELECT MAX(id) FROM DBPREFIXfilters`, nil, []any{&filterID}); err != nil {
return err
}
if fBanBoardID != nil {
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilter_boards(filter_id, board_id) VALUES(?,?)`, filterID, *fBanBoardID,
); err != nil {
return err
}
}
if fBanFingerprinter != nil {
field = *fBanFingerprinter
}
if field == "" {
field = "checksum"
}
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilter_conditions(filter_id,is_regex,search,field) VALUES(?,?,?,?)`, filterID, false, fBanChecksum, field,
); err != nil {
return err
}
}
return nil
}
func MigrateFilenameBans(db *gcsql.GCDB, ctx context.Context, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,filename,is_regex FROM DBPREFIXfilename_ban`)
if err != nil {
return err
}
var fnBanBoardID *int
var fnBanStaffID int
var fnBanStaffNote string
var fnBanIssuedAt time.Time
var fnBanFilename string
var fnBanIsRegex bool
var filterID int
for rows.Next() {
if err = rows.Scan(
&fnBanBoardID, &fnBanStaffID, &fnBanStaffNote, &fnBanIssuedAt, &fnBanFilename, &fnBanIsRegex,
); err != nil {
return err
}
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilters(staff_id, staff_note, issued_at, match_action, match_detail) VALUES(?,?,?,?,?)`,
fnBanStaffID, fnBanStaffNote, fnBanIssuedAt, "reject", "",
); err != nil {
return err
}
if err = db.QueryRowContextSQL(ctx, tx, `SELECT MAX(id) FROM DBPREFIXfilters`, nil, []any{&filterID}); err != nil {
return err
}
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilter_conditions(filter_id,is_regex,search,field) VALUES(?,?,?,?)`,
filterID, fnBanIsRegex, fnBanFilename, "filename",
); err != nil {
return err
}
}
return nil
}
func MigrateUsernameBans(db *gcsql.GCDB, ctx context.Context, tx *sql.Tx, cfg *config.SQLConfig) error {
rows, err := db.QueryContextSQL(ctx, nil, `SELECT board_id,staff_id,staff_note,issued_at,username,is_regex FROM DBPREFIXusername_ban`)
if err != nil {
return err
}
var unBanBoardID *int
var unBanStaffID int
var unBanStaffNote string
var unBanIssuedAt time.Time
var unBanUsername string
var unBanIsRegex bool
var filterID int
for rows.Next() {
if err = rows.Scan(
&unBanBoardID, &unBanStaffID, &unBanStaffNote, &unBanIssuedAt, &unBanUsername, &unBanIsRegex,
); err != nil {
return err
}
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilters(staff_id, staff_note, issued_at, match_action, match_detail) VALUES(?,?,?,?,?)`,
unBanStaffID, unBanStaffNote, unBanIssuedAt, "reject", "",
); err != nil {
return err
}
if err = db.QueryRowContextSQL(ctx, tx, `SELECT MAX(id) FROM DBPREFIXfilters`, nil, []any{&filterID}); err != nil {
return err
}
if _, err = db.ExecContextSQL(ctx, tx,
`INSERT INTO DBPREFIXfilter_conditions(filter_id,is_regex,search,field) VALUES(?,?,?,?)`,
filterID, unBanIsRegex, unBanUsername, "name",
); err != nil {
return err
}
}
return nil
}

43
cmd/gochan-migration/internal/common/sqlutil.go
View file

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"os"
"path"
"regexp"
"strings"
@ -49,6 +50,30 @@ func ColumnType(db *gcsql.GCDB, tx *sql.Tx, columnName string, tableName string,
return dataType, err
}
// TableExists returns true if the given table exists in the given database, and an error if one occured
func TableExists(db *gcsql.GCDB, tx *sql.Tx, tableName string, sqlConfig *config.SQLConfig) (bool, error) {
tableName = strings.ReplaceAll(tableName, "DBPREFIX", sqlConfig.DBprefix)
dbName := sqlConfig.DBname
var query string
var params []any
switch sqlConfig.DBtype {
case "mysql":
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?`
params = []any{dbName, tableName}
case "postgresql":
query = `SELECT COUNT(*) FROM information_schema.TABLES WHERE table_catalog = ? AND table_name = ?`
params = []any{dbName, tableName}
case "sqlite3":
query = `SELECT COUNT(*) FROM sqlite_master WHERE name = ? AND type = 'table'`
params = []any{tableName}
default:
return false, gcsql.ErrUnsupportedDB
}
var count int
err := db.QueryRowTxSQL(tx, query, params, []any{&count})
return count == 1, err
}
// IsStringType returns true if the given column data type is TEXT or VARCHAR
func IsStringType(dataType string) bool {
lower := strings.ToLower(dataType)
@ -75,13 +100,21 @@ func RunSQLFile(path string, db *gcsql.GCDB) error {
return nil
}
func InitDB(initFile string, db *gcsql.GCDB) error {
func getInitFilePath(initFile string) (string, error) {
filePath := gcutil.FindResource(initFile,
"/usr/local/share/gochan/"+initFile,
"/usr/share/gochan/"+initFile)
path.Join("./sql", initFile),
path.Join("/usr/local/share/gochan", initFile),
path.Join("/usr/share/gochan", initFile))
if filePath == "" {
return fmt.Errorf(
"SQL database initialization file (%s) missing. Please reinstall gochan-migration", initFile)
return "", fmt.Errorf("missing SQL database initialization file (%s), please reinstall gochan", initFile)
}
return filePath, nil
}
func InitDB(initFile string, db *gcsql.GCDB) error {
filePath, err := getInitFilePath(initFile)
if err != nil {
return err
}
return RunSQLFile(filePath, db)

45
cmd/gochan-migration/internal/gcupdate/gcupdate.go
View file

@ -11,14 +11,12 @@ import (
"github.com/gochan-org/gochan/pkg/gcutil"
)
const (
// if the database version is less than this, it is assumed to be out of date, and the schema needs to be adjusted
latestDatabaseVersion = 3
)
type GCDatabaseUpdater struct {
options *common.MigrationOptions
db *gcsql.GCDB
// if the database version is less than TargetDBVer, it is assumed to be out of date, and the schema needs to be adjusted.
// It is expected to be set by the build script
TargetDBVer int
}
func (dbu *GCDatabaseUpdater) Init(options *common.MigrationOptions) error {
@ -36,12 +34,12 @@ func (dbu *GCDatabaseUpdater) IsMigrated() (bool, error) {
if err != nil {
return false, err
}
if currentDatabaseVersion == latestDatabaseVersion {
if currentDatabaseVersion == dbu.TargetDBVer {
return true, nil
}
if currentDatabaseVersion > latestDatabaseVersion {
if currentDatabaseVersion > dbu.TargetDBVer {
return false, fmt.Errorf("database layout is ahead of current version (%d), target version: %d",
currentDatabaseVersion, latestDatabaseVersion)
currentDatabaseVersion, dbu.TargetDBVer)
}
return false, nil
}
@ -75,8 +73,37 @@ func (dbu *GCDatabaseUpdater) MigrateDB() (bool, error) {
return false, err
}
filterTableExists, err := common.TableExists(dbu.db, nil, "DBPREFIXfilters", &sqlConfig)
if err != nil {
return false, err
}
if !filterTableExists {
if err = common.AddFilterTables(dbu.db, ctx, tx, &sqlConfig); err != nil {
return false, err
}
if err = common.MigrateFileBans(dbu.db, ctx, tx, &sqlConfig); err != nil {
return false, err
}
if err = common.MigrateFilenameBans(dbu.db, ctx, tx, &sqlConfig); err != nil {
return false, err
}
if err = common.MigrateUsernameBans(dbu.db, ctx, tx, &sqlConfig); err != nil {
return false, err
}
if _, err = dbu.db.ExecContextSQL(ctx, tx, `DROP TABLE DBPREFIXfile_ban`); err != nil {
return false, err
}
if _, err = dbu.db.ExecContextSQL(ctx, tx, `DROP TABLE DBPREFIXfilename_ban`); err != nil {
return false, err
}
if _, err = dbu.db.ExecContextSQL(ctx, tx, `DROP TABLE DBPREFIXfile_ban`); err != nil {
return false, err
}
}
query := `UPDATE DBPREFIXdatabase_version SET version = ? WHERE component = 'gochan'`
_, err = dbu.db.ExecTxSQL(tx, query, latestDatabaseVersion)
_, err = dbu.db.ExecTxSQL(tx, query, dbu.TargetDBVer)
if err != nil {
return false, err
}

21
cmd/gochan-migration/main.go
View file

@ -4,6 +4,7 @@ import (
"flag"
"log"
"os"
"strconv"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/gcupdate"
@ -15,10 +16,7 @@ import (
const (
banner = `Welcome to the gochan database migration tool for gochan %s!
This migration tool is currently unstable, and will likely go through
several changes before it can be considered "stable", so make sure you check
the README and/or the -h command line flag before you use it.
Make sure you check the README and/or the -h command line flag, and back up your current database before you use it.
`
migrateCompleteTxt = `Database migration successful!
To migrate the uploads for each board, move or copy the uploads to /path/to/gochan/document/root/<boardname>/src/
@ -30,8 +28,9 @@ for the threads and board pages`
)
var (
versionStr string
migrator common.DBMigrator
versionStr string
migrator common.DBMigrator
dbVersionStr string
)
func cleanup() int {
@ -87,7 +86,13 @@ func main() {
log.Printf(banner, versionStr)
switch options.ChanType {
case "gcupdate":
migrator = &gcupdate.GCDatabaseUpdater{}
targetDBVer, err := strconv.Atoi(dbVersionStr)
if err != nil {
log.Fatalf("Invalid database version string %q, unable to parse to int", dbVersionStr)
}
migrator = &gcupdate.GCDatabaseUpdater{
TargetDBVer: targetDBVer,
}
case "pre2021":
migrator = &pre2021.Pre2021Migrator{}
case "kusabax":
@ -108,7 +113,7 @@ func main() {
if err != nil {
log.Fatalf("Failed to connect to the database: %s", err.Error())
}
if err = gcsql.CheckAndInitializeDatabase(sqlCfg.DBtype); err != nil {
if err = gcsql.CheckAndInitializeDatabase(sqlCfg.DBtype, dbVersionStr); err != nil {
log.Fatalf("Failed to initialize the database: %s", err.Error())
}
}

5
cmd/gochan/main.go
View file

@ -24,7 +24,8 @@ import (
)
var (
versionStr string
versionStr string
dbVersionStr string
)
func cleanup() {
@ -70,7 +71,7 @@ func main() {
Str("dbType", systemCritical.DBtype).
Msg("Connected to database")
if err = gcsql.CheckAndInitializeDatabase(systemCritical.DBtype); err != nil {
if err = gcsql.CheckAndInitializeDatabase(systemCritical.DBtype, dbVersionStr); err != nil {
cleanup()
gcutil.LogFatal().Err(err).Msg("Failed to initialize the database")
}