gochan/gochan
1
0
Fork 0
mirror of https://github.com/Eggbertx/gochan.git synced 2025-08-18 11:46:23 -07:00
Code Releases Activity

Start changing migrating to have "in-place" migration rename boards to be recreated and populated

Browse source
This commit is contained in:
Eggbertx 2025-02-02 15:26:33 -08:00
parent 9298aa42c3
commit cbd1dd8a99
12 changed files with 127 additions and 477 deletions
Download patch file Download diff file Expand all files Collapse all files

65
cmd/gochan-migration/internal/pre2021/announcements.go
View file

@ -8,63 +8,7 @@ import (
"github.com/gochan-org/gochan/pkg/gcsql"
)
func (m *Pre2021Migrator) migrateAnnouncementsInPlace() error {
errEv := common.LogError()
defer errEv.Discard()
if _, err := gcsql.ExecSQL(announcementsAlterStatement); err != nil {
errEv.Err(err).Caller().Msg("Failed to alter announcements table")
return err
}
var err error
m.migrationUser, err = m.getMigrationUser(errEv)
if err != nil {
errEv.Err(err).Caller().Msg("Failed to get migration user")
return err
}
rows, err := m.db.QuerySQL("SELECT poster FROM DBPREFIXannouncements")
if err != nil {
errEv.Err(err).Caller().Msg("Failed to get announcements")
return err
}
defer rows.Close()
var announcementPosters []string
for rows.Next() {
var poster string
if err = rows.Scan(&poster); err != nil {
errEv.Err(err).Caller().Msg("Failed to scan announcement row")
return err
}
announcementPosters = append(announcementPosters, poster)
}
if err = rows.Close(); err != nil {
errEv.Err(err).Caller().Msg("Failed to close announcement rows")
return err
}
for _, poster := range announcementPosters {
id, err := gcsql.GetStaffID(poster)
if errors.Is(err, gcsql.ErrUnrecognizedUsername) {
// user doesn't exist, use migration user
common.LogWarning().Str("staff", poster).Msg("Staff username not found in database")
id = m.migrationUser.ID
} else if err != nil {
errEv.Err(err).Caller().Str("staff", poster).Msg("Failed to get staff ID")
return err
}
if _, err = gcsql.ExecSQL("UPDATE DBPREFIXannouncements SET staff_id = ? WHERE poster = ?", id, poster); err != nil {
errEv.Err(err).Caller().Str("staff", poster).Msg("Failed to update announcement poster")
return err
}
}
return nil
}
func (m *Pre2021Migrator) migrateAnnouncementsToNewDB() error {
func (m *Pre2021Migrator) MigrateAnnouncements() error {
errEv := common.LogError()
defer errEv.Discard()
@ -112,10 +56,3 @@ func (m *Pre2021Migrator) migrateAnnouncementsToNewDB() error {
}
return nil
}
func (m *Pre2021Migrator) MigrateAnnouncements() error {
if m.IsMigratingInPlace() {
return m.migrateAnnouncementsInPlace()
}
return m.migrateAnnouncementsToNewDB()
}

27
cmd/gochan-migration/internal/pre2021/announcements_test.go
View file

@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert"
)
func TestMigrateAnnouncementsToNewDB(t *testing.T) {
func TestMigrateAnnouncements(t *testing.T) {
outDir := t.TempDir()
migrator := setupMigrationTest(t, outDir, false)
if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") {
@ -26,31 +26,6 @@ func TestMigrateAnnouncementsToNewDB(t *testing.T) {
t.FailNow()
}
validateAnnouncementMigration(t)
}
func TestMigrateAnnouncementsInPlace(t *testing.T) {
outDir := t.TempDir()
migrator := setupMigrationTest(t, outDir, true)
if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") {
t.FailNow()
}
if !assert.NoError(t, migrator.MigrateBoards()) {
t.FailNow()
}
if !assert.NoError(t, migrator.MigrateStaff()) {
t.FailNow()
}
if !assert.NoError(t, migrator.MigrateAnnouncements()) {
t.FailNow()
}
validateAnnouncementMigration(t)
}
func validateAnnouncementMigration(t *testing.T) {
var numAnnouncements int
assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXannouncements WHERE staff_id > 0", nil, []any{&numAnnouncements}))
assert.Equal(t, 2, numAnnouncements, "Expected to have two announcement")

28
cmd/gochan-migration/internal/pre2021/bans_test.go
View file

@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert"
)
func TestMigrateBansToNewDB(t *testing.T) {
func TestMigrateBans(t *testing.T) {
outDir := t.TempDir()
migrator := setupMigrationTest(t, outDir, false)
if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") {
@ -33,32 +33,6 @@ func TestMigrateBansToNewDB(t *testing.T) {
validateBanMigration(t)
}
func TestMigrateBansInPlace(t *testing.T) {
outDir := t.TempDir()
migrator := setupMigrationTest(t, outDir, true)
if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") {
t.FailNow()
}
if !assert.NoError(t, migrator.MigrateBoards()) {
t.FailNow()
}
if !assert.NoError(t, migrator.MigratePosts()) {
t.FailNow()
}
if !assert.NoError(t, migrator.MigrateStaff()) {
t.FailNow()
}
if !assert.NoError(t, migrator.MigrateBans()) {
t.FailNow()
}
validateBanMigration(t)
}
func validateBanMigration(t *testing.T) {
bans, err := gcsql.GetIPBans(0, 200, false)
if !assert.NoError(t, err) {

103
cmd/gochan-migration/internal/pre2021/boards.go
View file

@ -1,13 +1,9 @@
package pre2021
import (
"runtime/debug"
"strings"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
"github.com/gochan-org/gochan/pkg/config"
"github.com/gochan-org/gochan/pkg/gcsql"
"github.com/rs/zerolog"
)
type migrationBoard struct {
@ -21,41 +17,7 @@ type migrationSection struct {
gcsql.Section
}
func (m *Pre2021Migrator) migrateSectionsInPlace() error {
_, err := m.db.ExecSQL(`ALTER TABLE DBPREFIXsections RENAME COLUMN list_order TO position`)
if err != nil {
common.LogError().Caller().Msg("Failed to rename list_order column to position")
}
return err
}
func (m *Pre2021Migrator) migrateBoardsInPlace() error {
errEv := common.LogError()
defer errEv.Discard()
err := m.migrateSectionsInPlace()
if err != nil {
errEv.Err(err).Caller().Msg("Failed to migrate sections")
return err
}
for _, statement := range boardAlterStatements {
if strings.Contains(statement, "CONSTRAINT") && m.db.SQLDriver() == "sqlite3" {
// skip constraints in SQLite since they can't be added after table creation
continue
}
_, err = m.db.ExecSQL(statement)
if err != nil {
errEv.Err(err).Caller().
Str("statement", statement).
Msg("Failed to execute alter statement")
return err
}
}
return nil
}
func (m *Pre2021Migrator) migrateSectionsToNewDB() error {
func (m *Pre2021Migrator) migrateSections() error {
// creates sections in the new db if they don't exist, and also creates a migration section that
// boards will be set to, to be moved to the correct section by the admin after migration
errEv := common.LogError()
@ -74,6 +36,7 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error {
})
}
var sectionsToBeCreated []gcsql.Section
rows, err := m.db.QuerySQL(sectionsQuery)
if err != nil {
errEv.Err(err).Caller().Msg("Failed to query old database sections")
@ -99,33 +62,40 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error {
Int("oldSectionID", m.sections[s].oldID).
Str("sectionName", section.Name).
Str("sectionAbbreviation", section.Abbreviation).
Msg("Section already exists in new db, updating values")
if err = m.sections[s].UpdateValues(); err != nil {
errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to update pre-existing section values")
}
Msg("Section already exists in new db, values will be updated")
found = true
break
}
}
if !found {
migratedSection, err := gcsql.NewSection(section.Name, section.Abbreviation, section.Hidden, section.Position)
if err != nil {
errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to migrate section")
return err
}
m.sections = append(m.sections, migrationSection{
Section: *migratedSection,
})
sectionsToBeCreated = append(sectionsToBeCreated, section)
}
}
if err = rows.Close(); err != nil {
errEv.Caller().Msg("Failed to close section rows")
return err
}
for _, section := range sectionsToBeCreated {
migratedSection, err := gcsql.NewSection(section.Name, section.Abbreviation, section.Hidden, section.Position)
if err != nil {
errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to migrate section")
return err
}
m.sections = append(m.sections, migrationSection{
Section: *migratedSection,
})
}
for s, section := range m.sections {
if err = m.sections[s].UpdateValues(); err != nil {
errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to update pre-existing section values")
}
}
return nil
}
func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
func (m *Pre2021Migrator) MigrateBoards() error {
m.boards = nil
errEv := common.LogError()
defer errEv.Discard()
@ -137,7 +107,7 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
return nil
}
if err = m.migrateSectionsToNewDB(); err != nil {
if err = m.migrateSections(); err != nil {
// error should already be logged by migrateSectionsToNewDB
return err
}
@ -162,6 +132,7 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
return err
}
defer rows.Close()
var boardsTmp []migrationBoard
for rows.Next() {
var board migrationBoard
@ -176,8 +147,11 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
return err
}
board.MaxThreads = maxPages * config.GetBoardConfig(board.Dir).ThreadsPerPage
found := false
boardsTmp = append(boardsTmp, board)
}
for _, board := range boardsTmp {
found := false
for b, newBoard := range m.boards {
if newBoard.Dir == board.Dir {
m.boards[b].oldID = board.oldID
@ -229,24 +203,3 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
}
return nil
}
func (m *Pre2021Migrator) MigrateBoards() error {
defer func() {
if r := recover(); r != nil {
stackTrace := debug.Stack()
traceLines := strings.Split(string(stackTrace), "\n")
zlArr := zerolog.Arr()
for _, line := range traceLines {
zlArr.Str(line)
}
common.LogFatal().Caller().
Interface("recover", r).
Array("stackTrace", zlArr).
Msg("Recovered from panic in MigrateBoards")
}
}()
if m.IsMigratingInPlace() {
return m.migrateBoardsInPlace()
}
return m.migrateBoardsToNewDB()
}

15
cmd/gochan-migration/internal/pre2021/boards_test.go
View file

@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert"
)
func TestMigrateBoardsToNewDB(t *testing.T) {
func TestMigrateBoards(t *testing.T) {
outDir := t.TempDir()
migrator := setupMigrationTest(t, outDir, false)
if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") {
@ -27,19 +27,6 @@ func TestMigrateBoardsToNewDB(t *testing.T) {
validateBoardMigration(t)
}
func TestMigrateBoardsInPlace(t *testing.T) {
outDir := t.TempDir()
migrator := setupMigrationTest(t, outDir, true)
if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") {
t.FailNow()
}
if !assert.NoError(t, migrator.MigrateBoards()) {
t.FailNow()
}
validateBoardMigration(t)
}
func validateBoardMigration(t *testing.T) {
migratedBoards, err := gcsql.GetAllBoards(false)
if !assert.NoError(t, err) {

163
cmd/gochan-migration/internal/pre2021/posts.go
View file

@ -3,8 +3,6 @@ package pre2021
import (
"context"
"database/sql"
"os"
"strings"
"time"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
@ -35,13 +33,6 @@ type migrationPost struct {
oldParentID int
}
func (m *Pre2021Migrator) MigratePosts() error {
if m.IsMigratingInPlace() {
return m.migratePostsInPlace()
}
return m.migratePostsToNewDB()
}
func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *zerolog.Event) error {
var err error
@ -83,7 +74,7 @@ func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *ze
return nil
}
func (m *Pre2021Migrator) migratePostsToNewDB() error {
func (m *Pre2021Migrator) MigratePosts() error {
errEv := common.LogError()
defer errEv.Discard()
@ -195,155 +186,3 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error {
Msg("Migrated threads successfully")
return nil
}
func (m *Pre2021Migrator) migratePostsInPlace() error {
errEv := common.LogError()
defer errEv.Discard()
ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(m.config.DBTimeoutSeconds))
defer cancel()
ba, err := os.ReadFile(gcutil.FindResource("sql/initdb_" + m.db.SQLDriver() + ".sql"))
if err != nil {
errEv.Err(err).Caller().
Msg("Failed to read initdb SQL file")
return err
}
statements := strings.Split(string(ba), ";")
for _, statement := range statements {
statement = strings.TrimSpace(statement)
if strings.HasPrefix(statement, "CREATE TABLE DBPREFIXthreads") || strings.HasPrefix(statement, "CREATE TABLE DBPREFIXfiles") {
if _, err = m.db.ExecContextSQL(ctx, nil, statement); err != nil {
errEv.Err(err).Caller().Msg("Failed to create threads table")
return err
}
}
}
rows, err := m.db.QueryContextSQL(ctx, nil, threadsQuery+" AND parentid = 0")
if err != nil {
errEv.Err(err).Caller().
Msg("Failed to get threads")
return err
}
defer rows.Close()
var threads []migrationPost
for rows.Next() {
var post migrationPost
if err = rows.Scan(
&post.ID, &post.oldBoardID, &post.oldParentID, &post.Name, &post.Tripcode, &post.Email,
&post.Subject, &post.Message, &post.MessageRaw, &post.Password, &post.filename,
&post.filenameOriginal, &post.fileChecksum, &post.filesize, &post.imageW, &post.imageH,
&post.thumbW, &post.thumbH, &post.IP, &post.CreatedOn, &post.autosage,
&post.bumped, &post.stickied, &post.locked,
); err != nil {
errEv.Err(err).Caller().
Msg("Failed to scan thread")
return err
}
threads = append(threads, post)
}
if err = rows.Close(); err != nil {
errEv.Caller().Msg("Failed to close thread rows")
return err
}
for _, statements := range postAlterStatements {
if _, err = m.db.ExecContextSQL(ctx, nil, statements); err != nil {
errEv.Err(err).Caller().Msg("Failed to alter posts table")
return err
}
}
switch m.db.SQLDriver() {
case "mysql":
_, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts ADD COLUMN ip_new VARBINARY(16) NOT NULL")
case "postgres", "postgresql":
_, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts ADD COLUMN ip_new INET NOT NULL")
case "sqlite3":
_, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts ADD COLUMN ip_new VARCHAR(45) NOT NULL DEFAULT '0.0.0.0'")
}
if err != nil {
errEv.Err(err).Caller().Msg("Failed to update IP column")
return err
}
if _, err = m.db.ExecContextSQL(ctx, nil, "UPDATE DBPREFIXposts SET ip_new = IP_ATON"); err != nil {
errEv.Err(err).Caller().Msg("Failed to update IP column")
return err
}
if _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts RENAME COLUMN ip TO ip_old"); err != nil {
errEv.Err(err).Caller().Msg("Failed to rename old IP column")
return err
}
if _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts RENAME COLUMN ip_new TO ip"); err != nil {
errEv.Err(err).Caller().Msg("Failed to rename new IP column")
return err
}
for _, op := range threads {
if _, err = m.db.ExecContextSQL(ctx, nil,
`INSERT INTO DBPREFIXthreads(board_id,locked,stickied,anchored,cyclical,last_bump,is_deleted) VALUES(?,?,?,?,?,?,?)`,
op.oldBoardID, op.locked, op.stickied, op.autosage, false, op.bumped, false,
); err != nil {
errEv.Err(err).Caller().
Int("postID", op.ID).
Msg("Failed to insert thread")
return err
}
if err = m.db.QueryRowContextSQL(ctx, nil, "SELECT MAX(id) FROM DBPREFIXthreads", nil, []any{&op.ThreadID}); err != nil {
errEv.Err(err).Caller().
Int("postID", op.ID).
Msg("Failed to get thread ID")
return err
}
if _, err = m.db.ExecContextSQL(ctx, nil,
"UPDATE DBPREFIXposts SET thread_id = ? WHERE (id = ? and is_top_post) or thread_id = ?", op.ThreadID, op.oldID, op.oldID,
); err != nil {
errEv.Err(err).Caller().
Int("postID", op.ID).
Int("threadID", op.ThreadID).
Msg("Failed to set thread ID")
return err
}
}
if rows, err = m.db.QueryContextSQL(ctx, nil,
"SELECT id,filename,filename_original,file_checksum,filesize,image_w,image_h,thumb_w,thumb_h FROM DBPREFIXposts WHERE filename <> ''",
); err != nil {
errEv.Err(err).Caller().Msg("Failed to get uploads")
return err
}
defer rows.Close()
var uploads []gcsql.Upload
for rows.Next() {
var upload gcsql.Upload
if err = rows.Scan(&upload.PostID, &upload.Filename, &upload.OriginalFilename, &upload.Checksum, &upload.FileSize, &upload.Width,
&upload.Height, &upload.ThumbnailWidth, &upload.ThumbnailHeight,
); err != nil {
errEv.Err(err).Caller().Msg("Failed to scan upload")
return err
}
uploads = append(uploads, upload)
}
if err = rows.Close(); err != nil {
errEv.Caller().Msg("Failed to close upload rows")
return err
}
for _, upload := range uploads {
if _, err = m.db.ExecContextSQL(ctx, nil,
`INSERT INTO DBPREFIXfiles(post_id,file_order,filename,original_filename,checksum,file_size,width,height,thumbnail_width,thumbnail_height,is_spoilered) VALUES
(?,0,?,?,?,?,?,?,?,?,0)`,
upload.PostID, upload.Filename, upload.OriginalFilename, upload.Checksum, upload.FileSize, upload.Width, upload.Height,
upload.ThumbnailWidth, upload.ThumbnailHeight,
); err != nil {
errEv.Err(err).Caller().
Int("postID", upload.PostID).
Msg("Failed to insert upload")
return err
}
}
return nil
}

19
cmd/gochan-migration/internal/pre2021/posts_test.go
View file

@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert"
)
func TestMigratePostsToNewDB(t *testing.T) {
func TestMigratePosts(t *testing.T) {
outDir := t.TempDir()
migrator := setupMigrationTest(t, outDir, false)
if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") {
@ -30,23 +30,6 @@ func TestMigratePostsToNewDB(t *testing.T) {
validatePostMigration(t)
}
func TestMigratePostsInPlace(t *testing.T) {
outDir := t.TempDir()
migrator := setupMigrationTest(t, outDir, true)
if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") {
t.FailNow()
}
if !assert.NoError(t, migrator.MigrateBoards()) {
t.FailNow()
}
if !assert.NoError(t, migrator.MigratePosts()) {
t.FailNow()
}
validatePostMigration(t)
}
func validatePostMigration(t *testing.T) {
var numThreads int
if !assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numThreads}), "Failed to get number of threads") {

47
cmd/gochan-migration/internal/pre2021/pre2021.go
View file

@ -4,6 +4,7 @@ package pre2021
import (
"context"
"encoding/json"
"fmt"
"os"
"time"
@ -25,6 +26,7 @@ type Pre2021Migrator struct {
migrationUser *gcsql.Staff
boards []migrationBoard
sections []migrationSection
staff []migrationStaff
}
// IsMigratingInPlace implements common.DBMigrator.
@ -45,7 +47,7 @@ func (m *Pre2021Migrator) readConfig() error {
func (m *Pre2021Migrator) Init(options *common.MigrationOptions) error {
m.options = options
var err error
m.config.SQLConfig = config.GetSQLConfig()
if err = m.readConfig(); err != nil {
return err
}
@ -66,6 +68,43 @@ func (m *Pre2021Migrator) IsMigrated() (bool, error) {
return common.TableExists(ctx, m.db, nil, "DBPREFIXdatabase_version", &sqlConfig)
}
func (m *Pre2021Migrator) renameTablesForInPlace() error {
var err error
errEv := common.LogError()
defer errEv.Discard()
if _, err = m.db.ExecSQL("DROP TABLE DBPREFIXinfo"); err != nil {
errEv.Err(err).Caller().Msg("Error dropping info table")
return err
}
for _, table := range renameTables {
if _, err = m.db.ExecSQL(fmt.Sprintf(renameTableStatementTemplate, table, table)); err != nil {
errEv.Caller().Err(err).
Str("table", table).
Msg("Error renaming table")
return err
}
}
if err = gcsql.CheckAndInitializeDatabase(m.config.DBtype, "4"); err != nil {
errEv.Caller().Err(err).Msg("Error checking and initializing database")
return err
}
if err = m.Close(); err != nil {
errEv.Err(err).Caller().Msg("Error closing database")
return err
}
m.config.SQLConfig.DBprefix = "_tmp_" + m.config.DBprefix
m.db, err = gcsql.Open(&m.config.SQLConfig)
if err != nil {
errEv.Err(err).Caller().Msg("Error reopening database with new prefix")
return err
}
common.LogInfo().Msg("Renamed tables for in-place migration")
return err
}
func (m *Pre2021Migrator) MigrateDB() (bool, error) {
errEv := common.LogError()
defer errEv.Discard()
@ -78,6 +117,12 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) {
return true, nil
}
if m.IsMigratingInPlace() {
if err = m.renameTablesForInPlace(); err != nil {
return false, err
}
}
if err := m.MigrateBoards(); err != nil {
return false, err
}

8
cmd/gochan-migration/internal/pre2021/pre2021_test.go
View file

@ -97,7 +97,9 @@ func TestPre2021MigrationToNewDB(t *testing.T) {
t.FailNow()
}
migrated, err := migrator.MigrateDB()
assert.NoError(t, err)
if !assert.NoError(t, err) {
t.FailNow()
}
assert.False(t, migrated)
validateBoardMigration(t)
@ -113,7 +115,9 @@ func TestPre2021MigrationInPlace(t *testing.T) {
t.FailNow()
}
migrated, err := migrator.MigrateDB()
assert.NoError(t, err)
if !assert.NoError(t, err) {
t.FailNow()
}
assert.False(t, migrated)
validateBoardMigration(t)

26
cmd/gochan-migration/internal/pre2021/queries.go
View file

@ -14,14 +14,14 @@ bumped, stickied, locked FROM DBPREFIXposts WHERE deleted_timestamp IS NULL`
threadsQuery = postsQuery + " AND parentid = 0"
staffQuery = `SELECT username, rank, boards, added_on, last_active FROM DBPREFIXstaff`
staffQuery = `SELECT id, username, rank, boards, added_on, last_active FROM DBPREFIXstaff`
bansQuery = `SELECT id, allow_read, COALESCE(ip, '') as ip, name, name_is_regex, filename, file_checksum, boards, staff,
timestamp, expires, permaban, reason, type, staff_note, appeal_at, can_appeal FROM DBPREFIXbanlist`
announcementsQuery = "SELECT id, subject, message, poster, timestamp FROM DBPREFIXannouncements"
announcementsAlterStatement = "ALTER TABLE DBPREFIXannouncements ADD COLUMN staff_id INT NOT NULL DEFAULT 1"
renameTableStatementTemplate = "ALTER TABLE %s RENAME TO _tmp_%s"
)
var (
@ -41,22 +41,10 @@ var (
"ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_dir_unique UNIQUE (dir)",
"ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_uri_unique UNIQUE (uri)",
}
postAlterStatements = []string{
"ALTER TABLE DBPREFIXposts RENAME COLUMN parentid TO thread_id",
"ALTER TABLE DBPREFIXposts RENAME COLUMN timestamp TO created_on",
"ALTER TABLE DBPREFIXposts RENAME COLUMN deleted_timestamp TO deleted_at",
// "ALTER TABLE DBPREFIXposts RENAME COLUMN ip TO ip_old",
"ALTER TABLE DBPREFIXposts ADD COLUMN is_top_post BOOL NOT NULL DEFAULT FALSE",
"ALTER TABLE DBPREFIXposts ADD COLUMN is_role_signature BOOL NOT NULL DEFAULT FALSE",
"ALTER TABLE DBPREFIXposts ADD COLUMN is_deleted BOOL NOT NULL DEFAULT FALSE",
"ALTER TABLE DBPREFIXposts ADD COLUMN banned_message TEXT",
"ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''",
"ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''",
"UPDATE DBPREFIXposts SET is_top_post = TRUE WHERE thread_id = 0",
}
staffAlterStatements = []string{
"ALTER TABLE DBPREFIXstaff RENAME COLUMN rank TO global_rank",
"ALTER TABLE DBPREFIXstaff RENAME COLUMN last_active TO last_login",
"ALTER TABLE DBPREFIXstaff ADD COLUMN is_active BOOL NOT NULL DEFAULT TRUE",
// tables to be renamed to _tmp_DBPREFIX* to work around SQLite's lack of support for changing/removing columns
renameTables = []string{
"DBPREFIXannouncements", "DBPREFIXappeals", "DBPREFIXbanlist", "DBPREFIXboards", "DBPREFIXembeds", "DBPREFIXlinks",
"DBPREFIXposts", "DBPREFIXreports", "DBPREFIXsections", "DBPREFIXsessions", "DBPREFIXstaff", "DBPREFIXwordfilters",
}
)

88
cmd/gochan-migration/internal/pre2021/staff.go
View file

@ -11,23 +11,10 @@ import (
"github.com/rs/zerolog"
)
func (m *Pre2021Migrator) migrateStaffInPlace() error {
errEv := common.LogError()
defer errEv.Discard()
for _, stmt := range staffAlterStatements {
if _, err := gcsql.ExecSQL(stmt); err != nil {
errEv.Err(err).Caller().Msg("Failed to alter staff table")
return err
}
}
_, err := m.getMigrationUser(errEv)
if err != nil {
return err
}
return nil
type migrationStaff struct {
gcsql.Staff
boards string
oldID int
}
func (m *Pre2021Migrator) getMigrationUser(errEv *zerolog.Event) (*gcsql.Staff, error) {
@ -36,7 +23,7 @@ func (m *Pre2021Migrator) getMigrationUser(errEv *zerolog.Event) (*gcsql.Staff,
}
user := &gcsql.Staff{
Username: "pre2021-migration" + gcutil.RandomString(15),
Username: "pre2021-migration" + gcutil.RandomString(8),
AddedOn: time.Now(),
}
_, err := gcsql.ExecSQL("INSERT INTO DBPREFIXstaff(username,password_checksum,global_rank,is_active) values(?,'',0,0)", user.Username)
@ -50,10 +37,11 @@ func (m *Pre2021Migrator) getMigrationUser(errEv *zerolog.Event) (*gcsql.Staff,
return nil, err
}
m.migrationUser = user
m.staff = append(m.staff, migrationStaff{Staff: *user})
return user, nil
}
func (m *Pre2021Migrator) migrateStaffToNewDB() error {
func (m *Pre2021Migrator) MigrateStaff() error {
errEv := common.LogError()
defer errEv.Discard()
@ -70,56 +58,53 @@ func (m *Pre2021Migrator) migrateStaffToNewDB() error {
defer rows.Close()
for rows.Next() {
var username string
var rank int
var boards string
var addedOn, lastActive time.Time
if err = rows.Scan(&username, &rank, &boards, &addedOn, &lastActive); err != nil {
var staff migrationStaff
if err = rows.Scan(&staff.oldID, &staff.Username, &staff.Rank, &staff.boards, &staff.AddedOn, &staff.LastLogin); err != nil {
errEv.Err(err).Caller().Msg("Failed to scan staff row")
return err
}
_, err = gcsql.GetStaffByUsername(username, false)
m.staff = append(m.staff, staff)
}
for _, staff := range m.staff {
newStaff, err := gcsql.GetStaffByUsername(staff.Username, false)
if err == nil {
// found staff
gcutil.LogInfo().Str("username", username).Int("rank", rank).Msg("Found matching staff account")
}
if errors.Is(err, gcsql.ErrUnrecognizedUsername) {
gcutil.LogInfo().Str("username", staff.Username).Int("rank", staff.Rank).Msg("Found matching staff account")
staff.ID = newStaff.ID
} else if errors.Is(err, gcsql.ErrUnrecognizedUsername) {
// staff doesn't exist, create it (with invalid checksum to be updated by the admin)
if _, err2 := gcsql.ExecSQL(
if _, err := gcsql.ExecSQL(
"INSERT INTO DBPREFIXstaff(username,password_checksum,global_rank,added_on,last_login,is_active) values(?,'',?,?,?,1)",
username, rank, addedOn, lastActive,
); err2 != nil {
errEv.Err(err2).Caller().
Str("username", username).Int("rank", rank).
Msg("Failed to migrate staff account")
staff.Username, staff.Rank, staff.AddedOn, staff.LastLogin,
); err != nil {
errEv.Err(err).Caller().Str("username", staff.Username).Int("rank", staff.Rank).Msg("Failed to migrate staff account")
return err
}
gcutil.LogInfo().Str("username", username).Int("rank", rank).Msg("Successfully migrated staff account")
} else if err != nil {
errEv.Err(err).Caller().Str("username", username).Msg("Failed to get staff account info")
if staff.ID, err = gcsql.GetStaffID(staff.Username); err != nil {
errEv.Err(err).Caller().Str("username", staff.Username).Msg("Failed to get staff account ID")
return err
}
gcutil.LogInfo().Str("username", staff.Username).Int("rank", staff.Rank).Msg("Successfully migrated staff account")
} else {
errEv.Err(err).Caller().Str("username", staff.Username).Msg("Failed to get staff account info")
return err
}
staffID, err := gcsql.GetStaffID(username)
if err != nil {
errEv.Err(err).Caller().Str("username", username).Msg("Failed to get staff account ID")
return err
}
if boards != "" && boards != "*" {
boardsArr := strings.Split(boards, ",")
if staff.boards != "" && staff.boards != "*" {
boardsArr := strings.Split(staff.boards, ",")
for _, board := range boardsArr {
board = strings.TrimSpace(board)
boardID, err := gcsql.GetBoardIDFromDir(board)
if err != nil {
errEv.Err(err).Caller().
Str("username", username).
Str("username", staff.Username).
Str("board", board).
Msg("Failed to get board ID")
return err
}
if _, err = gcsql.ExecSQL("INSERT INTO DBPREFIXboard_staff(board_id,staff_id) VALUES(?,?)", boardID, staffID); err != nil {
if _, err = gcsql.ExecSQL("INSERT INTO DBPREFIXboard_staff(board_id,staff_id) VALUES(?,?)", boardID, staff.ID); err != nil {
errEv.Err(err).Caller().
Str("username", username).
Str("username", staff.Username).
Str("board", board).
Msg("Failed to apply staff board info")
return err
@ -134,10 +119,3 @@ func (m *Pre2021Migrator) migrateStaffToNewDB() error {
}
return nil
}
func (m *Pre2021Migrator) MigrateStaff() error {
if m.IsMigratingInPlace() {
return m.migrateStaffInPlace()
}
return m.migrateStaffToNewDB()
}

15
cmd/gochan-migration/internal/pre2021/staff_test.go
View file

@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert"
)
func TestMigrateStaffToNewDB(t *testing.T) {
func TestMigrateStaff(t *testing.T) {
outDir := t.TempDir()
migrator := setupMigrationTest(t, outDir, false)
if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") {
@ -24,19 +24,6 @@ func TestMigrateStaffToNewDB(t *testing.T) {
validateStaffMigration(t)
}
func TestMigrateStaffInPlace(t *testing.T) {
outDir := t.TempDir()
migrator := setupMigrationTest(t, outDir, true)
if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") {
t.FailNow()
}
if !assert.NoError(t, migrator.MigrateStaff()) {
t.FailNow()
}
validateStaffMigration(t)
}
func validateStaffMigration(t *testing.T) {
migratedAdmin, err := gcsql.GetStaffByUsername("migratedadmin", true)
if !assert.NoError(t, err) {