Fully migrate and update board and section data to new DB for pre-2021

cmd/gochan-migration/internal/pre2021/boards.go

@@ -12,6 +12,7 @@ import (
 
 type migrationBoard struct {
 	oldSectionID int
+	oldID        int
 	gcsql.Board
 }
 
@@ -71,8 +72,20 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error {
 		var found bool
 		for s, newSection := range m.sections {
 			if section.Name == newSection.Name {
+				// section already exists, update values
 				m.sections[s].oldID = section.ID
-
+				m.sections[s].Abbreviation = section.Abbreviation
+				m.sections[s].Hidden = section.Hidden
+				m.sections[s].Position = section.Position
+				common.LogInfo().
+					Int("sectionID", section.ID).
+					Int("oldSectionID", m.sections[s].oldID).
+					Str("sectionName", section.Name).
+					Str("sectionAbbreviation", section.Abbreviation).
+					Msg("Section already exists in new db, updating values")
+				if err = m.sections[s].UpdateValues(); err != nil {
+					errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to update pre-existing section values")
+				}
 				found = true
 				break
 			}
@@ -96,9 +109,7 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error {
 }
 
 func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
-	if m.boards == nil {
-		m.boards = make(map[string]migrationBoard)
-	}
+	m.boards = nil
 	errEv := common.LogError()
 	defer errEv.Discard()
 
@@ -110,10 +121,22 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
 	}
 
 	if err = m.migrateSectionsToNewDB(); err != nil {
-		// error already logged
+		// error should already be logged by migrateSectionsToNewDB
 		return err
 	}
 
+	allBoards, err := gcsql.GetAllBoards(false)
+	if err != nil {
+		errEv.Err(err).Caller().Msg("Failed to get all boards from new db")
+		return err
+	}
+	for _, board := range allBoards {
+		m.boards = append(m.boards, migrationBoard{
+			oldSectionID: -1,
+			Board:        board,
+		})
+	}
+
 	// get boards from old db
 	rows, err := m.db.QuerySQL(boardsQuery)
 	if err != nil {
@@ -126,44 +149,49 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
 		var board migrationBoard
 		var maxPages int
 		if err = rows.Scan(
-			&board.ID, &board.NavbarPosition, &board.Dir, &board.Title, &board.Subtitle,
-			&board.Description, &board.SectionID, &board.MaxFilesize, &maxPages, &board.DefaultStyle, &board.Locked,
-			&board.CreatedAt, &board.AnonymousName, &board.ForceAnonymous, &board.AutosageAfter, &board.NoImagesAfter,
-			&board.MaxMessageLength, &board.AllowEmbeds, &board.RedirectToThread, &board.RequireFile, &board.EnableCatalog,
+			&board.ID, &board.NavbarPosition, &board.Dir, &board.Title, &board.Subtitle, &board.Description,
+			&board.SectionID, &board.MaxFilesize, &maxPages, &board.DefaultStyle, &board.Locked, &board.CreatedAt,
+			&board.AnonymousName, &board.ForceAnonymous, &board.AutosageAfter, &board.NoImagesAfter, &board.MaxMessageLength,
+			&board.AllowEmbeds, &board.RedirectToThread, &board.RequireFile, &board.EnableCatalog,
 		); err != nil {
 			errEv.Err(err).Caller().Msg("Failed to scan row into board")
 			return err
 		}
 		board.MaxThreads = maxPages * config.GetBoardConfig(board.Dir).ThreadsPerPage
 		found := false
-		for _, newBoard := range gcsql.AllBoards {
-			if _, ok := m.boards[board.Dir]; !ok {
-				m.boards[board.Dir] = board
-			}
+
+		for b, newBoard := range m.boards {
 			if newBoard.Dir == board.Dir {
-				common.LogWarning().Str("board", board.Dir).Msg("Board already exists in new db, moving on")
+				m.boards[b].oldID = board.ID
+				m.boards[b].oldSectionID = board.SectionID
+				common.LogInfo().Str("board", board.Dir).Msg("Board already exists in new db, updating values")
+				// don't update other values in the array since they don't affect migrating threads or posts
+				if _, err = gcsql.ExecSQL(`UPDATE DBPREFIXboards
+					SET uri = ?, navbar_position = ?, title = ?, subtitle = ?, description = ?,
+					max_file_size = ?, max_threads = ?, default_style = ?, locked = ?,
+					anonymous_name = ?, force_anonymous = ?, autosage_after = ?, no_images_after = ?, max_message_length = ?,
+					min_message_length = ?, allow_embeds = ?, redirect_to_thread = ?, require_file = ?, enable_catalog = ?
+					WHERE id = ?`,
+					board.Dir, board.NavbarPosition, board.Title, board.Subtitle, board.Description,
+					board.MaxFilesize, board.MaxThreads, board.DefaultStyle, board.Locked,
+					board.AnonymousName, board.ForceAnonymous, board.AutosageAfter, board.NoImagesAfter, board.MaxMessageLength,
+					board.MinMessageLength, board.AllowEmbeds, board.RedirectToThread, board.RequireFile, board.EnableCatalog,
+					newBoard.ID); err != nil {
+					errEv.Err(err).Caller().Str("board", board.Dir).Msg("Failed to update board values")
+					return err
+				}
 				found = true
 				break
 			}
 		}
-		if !found {
-			for _, section := range m.sections {
-				if section.oldID == board.oldSectionID {
-					board.SectionID = section.ID
-					break
-				}
-			}
-		}
 
-		m.boards[board.Dir] = board
 		if found {
-			// TODO: update board title, subtitle, section etc. in new db
 			continue
 		}
 
 		// create new board using the board data from the old db
 		// omitting things like ID and creation date since we don't really care
-		if err = gcsql.CreateBoard(&board.Board, false); err != nil {
+		if err = gcsql.CreateBoard(&board.Board, board.IsHidden(false)); err != nil {
 			errEv.Err(err).Caller().Str("board", board.Dir).Msg("Failed to create board")
 			return err
 		}

cmd/gochan-migration/internal/pre2021/pre2021.go

@@ -22,11 +22,9 @@ type Pre2021Migrator struct {
 	options *common.MigrationOptions
 	config  Pre2021Config
 
-	migrationSectionID int
-	posts              []postTable
-	boards             map[string]migrationBoard
-	sections           []migrationSection
-	threads            map[int]gcsql.Thread // old thread id (previously stored in posts ) to new thread id (threads.id)
+	posts    []postTable
+	boards   []migrationBoard
+	sections []migrationSection
 }
 
 // IsMigratingInPlace implements common.DBMigrator.

cmd/gochan-migration/internal/pre2021/sqlite3_test.go

@@ -14,7 +14,7 @@ const (
 	sqlite3DBPath = "tools/gochan-pre2021.sqlite3db" // relative to gochan project root
 )
 
-func setupMigrationTest(t *testing.T, outDir string) *Pre2021Migrator {
+func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre2021Migrator {
 	dir, err := testutil.GoToGochanRoot(t)
 	if !assert.NoError(t, err) {
 		t.FailNow()
@@ -41,7 +41,6 @@ func setupMigrationTest(t *testing.T, outDir string) *Pre2021Migrator {
 	}
 	migrator.db = db
 	migratedDBPath := path.Join(outDir, "gochan-migrated.sqlite3db")
-	t.Log("Migrated DB path:", migratedDBPath)
 
 	config.SetTestDBConfig("sqlite3", migratedDBPath, path.Base(migratedDBPath), "gochan", "password", "gc_")
 	sqlConfig := config.GetSQLConfig()
@@ -59,7 +58,7 @@ func setupMigrationTest(t *testing.T, outDir string) *Pre2021Migrator {
 
 func TestMigrateBoardsToNewDB(t *testing.T) {
 	outDir := t.TempDir()
-	migrator := setupMigrationTest(t, outDir)
+	migrator := setupMigrationTest(t, outDir, false)
 	assert.NoError(t, gcsql.ResetBoardSectionArrays())
 
 	numBoards := len(gcsql.AllBoards)
@@ -68,7 +67,7 @@ func TestMigrateBoardsToNewDB(t *testing.T) {
 	assert.Equal(t, 1, numBoards, "Expected to have 1 board pre-migration (/test/ is automatically created during provisioning)")
 	assert.Equal(t, 1, numSections, "Expected to have 1 section pre-migration (Main is automatically created during provisioning)")
 
-	assert.NoError(t, migrator.migrateBoardsToNewDB())
+	assert.NoError(t, migrator.MigrateBoards())
 
 	migratedBoards, err := gcsql.GetAllBoards(false)
 	if !assert.NoError(t, err) {
@@ -82,20 +81,29 @@ func TestMigrateBoardsToNewDB(t *testing.T) {
 	assert.Equal(t, len(migratedBoards), 2, "Expected updated boards list to have two boards")
 	assert.Equal(t, len(migratedSections), 2, "Expected updated sections list to have two sections")
 
+	// Test migrated sections
+	mainSection, err := gcsql.GetSectionFromName("Main")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	assert.Equal(t, "mainmigration", mainSection.Abbreviation, "Expected Main section to have updated abbreviation name 'mainmigration'")
+
+	// Test migrated boards
+	testBoard, err := gcsql.GetBoardFromDir("test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	assert.Equal(t, "Testing Board", testBoard.Title)
+	assert.Equal(t, "Board for testing pre-2021 migration", testBoard.Subtitle)
+	testBoardSection, err := gcsql.GetSectionFromID(testBoard.SectionID)
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	assert.Equal(t, "Main", testBoardSection.Name, "Expected /test/ board to be in Main section")
+
 	hiddenBoard, err := gcsql.GetBoardFromDir("hidden")
 	if !assert.NoError(t, err) {
 		t.FailNow()
 	}
-	t.Logf("Hidden board section ID: %d", hiddenBoard.SectionID)
-
-	t.Log("Number of sections:", len(migratedSections))
-	for _, section := range migratedSections {
-		t.Logf("Section ID %d: %#v", section.ID, section)
-	}
-	hiddenSection, err := gcsql.GetSectionFromID(hiddenBoard.SectionID)
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	assert.Equal(t, "Hidden section", hiddenSection.Name, "Expected /hidden/ board's section to have name 'Hidden'")
-	assert.True(t, hiddenSection.Hidden, "Expected Hidden section to be hidden")
+	assert.Equal(t, "Hidden Board", hiddenBoard.Title)
 }