From fd6b9258957980e5988afefe1d304c12323a0450 Mon Sep 17 00:00:00 2001 From: onihilist Date: Wed, 18 Dec 2024 17:08:42 +0100 Subject: [PATCH 001/122] Init commit --- pkg/config/config.go | 98 ++++++++++++++++++++++++++++++++++ pkg/server/.vscode/launch.json | 7 +++ pkg/server/server_test.go | 55 +++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 pkg/server/.vscode/launch.json create mode 100644 pkg/server/server_test.go diff --git a/pkg/config/config.go b/pkg/config/config.go index e0a17981..89fd4665 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,6 +38,104 @@ type GochanConfig struct { testing bool } +func SetMockConfig() { + cfg = &GochanConfig{ + SystemCriticalConfig: SystemCriticalConfig{ + ListenIP: "127.0.0.1", + Port: 8080, + UseFastCGI: false, + DocumentRoot: "/var/www/html", + TemplateDir: "/var/www/html/templates", + LogDir: "/var/log/gochan", + Plugins: []string{}, + PluginSettings: map[string]any{ + "examplePlugin": true, + }, + SiteHeaderURL: "http://example.com", + WebRoot: "/", + SiteDomain: "example.com", + Verbose: true, + RandomSeed: "testseed", + }, + SiteConfig: SiteConfig{ + FirstPage: []string{"index"}, + Username: "admin", + CookieMaxAge: "1h", + StaffSessionDuration: "2h", + Lockdown: false, + LockdownMessage: "", + SiteName: "Test Site", + SiteSlogan: "A testing site", + Modboard: "mod", + MaxRecentPosts: 10, + RecentPostsWithNoFile: false, + EnableAppeals: true, + MinifyHTML: false, + MinifyJS: false, + GeoIPType: "none", + GeoIPOptions: make(map[string]any), + Captcha: CaptchaConfig{ + Type: "none", + OnlyNeededForThreads: false, + SiteKey: "", + AccountSecret: "", + }, + FingerprintVideoThumbnails: false, + FingerprintHashLength: 16, + }, + BoardConfig: BoardConfig{ + InheritGlobalStyles: true, + Styles: []Style{}, + DefaultStyle: "default", + Banners: []PageBanner{}, + PostConfig: PostConfig{ + MaxLineLength: 1000, + ReservedTrips: []string{}, + ThreadsPerPage: 10, + RepliesOnBoardPage: 5, + StickyRepliesOnBoardPage: 2, + NewThreadsRequireUpload: false, + CyclicalThreadNumPosts: 100, + BanColors: []string{"#FF0000"}, + BanMessage: "You are banned!", + EmbedWidth: 640, + EmbedHeight: 360, + EnableEmbeds: true, + ImagesOpenNewTab: false, + NewTabOnOutlinks: true, + DisableBBcode: false, + }, + UploadConfig: UploadConfig{ + RejectDuplicateImages: true, + ThumbWidth: 150, + ThumbHeight: 150, + ThumbWidthReply: 100, + ThumbHeightReply: 100, + ThumbWidthCatalog: 200, + ThumbHeightCatalog: 200, + AllowOtherExtensions: map[string]string{"pdf": "application/pdf"}, + StripImageMetadata: "none", + ExiftoolPath: "", + }, + DateTimeFormat: "2006-01-02 15:04:05", + ShowPosterID: true, + EnableSpoileredImages: false, + EnableSpoileredThreads: false, + Worksafe: false, + ThreadPage: 1, + Cooldowns: BoardCooldowns{NewThread: 60, Reply: 30, ImageReply: 15}, + RenderURLsAsLinks: true, + ThreadsPerPage: 10, + EnableGeoIP: false, + EnableNoFlag: false, + CustomFlags: []geoip.Country{}, + isGlobal: true, + }, + jsonLocation: "test_config.json", + testing: true, + } +} + // ValidateValues checks to make sure that the configuration options are usable // (e.g., ListenIP is a valid IP address, Port isn't a negative number, etc) func (gcfg *GochanConfig) ValidateValues() error { diff --git a/pkg/server/.vscode/launch.json b/pkg/server/.vscode/launch.json new file mode 100644 index 00000000..5c7247b4 --- /dev/null +++ b/pkg/server/.vscode/launch.json @@ -0,0 +1,7 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [] +} \ No newline at end of file diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go new file mode 100644 index 00000000..25fe7024 --- /dev/null +++ b/pkg/server/server_test.go @@ -0,0 +1,55 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gochan-org/gochan/pkg/config" +) + +func TestServeJSON(t *testing.T) { + // Set up a mock configuration + config.SetMockConfig() + + writer := httptest.NewRecorder() + data := map[string]interface{}{ + "status": "success", + "postID": "777", + "srcBoard": "srcBoard", + "destBoard": "destBoard", + } + + ServeJSON(writer, data) + + // Check the status code + if writer.Code != http.StatusOK { + t.Errorf("expected status code %d, got %d", http.StatusOK, writer.Code) + } + + // Check the content type + if writer.Header().Get("Content-Type") != "application/json" { + t.Errorf("expected content type application/json, got %s", writer.Header().Get("Content-Type")) + } + + // Check the response body + var response map[string]interface{} + if err := json.Unmarshal(writer.Body.Bytes(), &response); err != nil { + t.Fatalf("failed to unmarshal response: %v", err) + } + + // Debugging output + for key, value := range response { + t.Logf("Response key: %s, value: %v", key, value) + } + + for key, expectedValue := range data { + actualValue := response[key] + if actualValue != expectedValue { + t.Errorf("expected %s to be %v, got %v", key, expectedValue, actualValue) + } else { + t.Logf("Match for %s: expected %v, got %v", key, expectedValue, actualValue) + } + } +} From 5cfa7628d931309734f0fc6b0c97101ac609f806 Mon Sep 17 00:00:00 2001 From: onihilist Date: Thu, 19 Dec 2024 10:34:40 +0100 Subject: [PATCH 002/122] Draft : Server tests --- pkg/server/server_test.go | 88 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 25fe7024..4f72dc08 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/gochan-org/gochan/pkg/config" + "github.com/stretchr/testify/assert" ) func TestServeJSON(t *testing.T) { @@ -53,3 +54,90 @@ func TestServeJSON(t *testing.T) { } } } + +func TestServeErrorPage(t *testing.T) { + // Set up a mock configuration + config.SetMockConfig() + + // Set writer and error string message + writer := httptest.NewRecorder() + err := "Unexpected error has occurred." + + ServeErrorPage(writer, err) + + body := writer.Body.String() + t.Log("=============") + t.Log("Response Body:", body) + t.Log("=============") + + // Check response code & content-type + assert.Equal(t, http.StatusOK, writer.Code) + assert.Equal(t, "text/html; charset=utf-8", writer.Header().Get("Content-Type")) + + // Check the response body for the error message + //assert.Contains(t, body, err) // Check if the body contains the error message + //assert.Contains(t, body, "Error") // Check if the body contains the error title or header +} + +func TestServeError(t *testing.T) { + // Set up a mock configuration + config.SetMockConfig() + + tests := []struct { + name string + err string + wantsJSON bool + data map[string]interface{} + expected string + }{ + { + name: "JSON response with error", + err: "some error occurred", + wantsJSON: true, + data: nil, + expected: `some error occurred`, + }, + { + name: "JSON response with existing data", + err: "another error occurred", + wantsJSON: true, + data: map[string]interface{}{"info": "some info"}, + expected: `another error occurred`, + }, + { + name: "Non-JSON response", + err: "page not found", + wantsJSON: false, + data: nil, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a response recorder to capture the response + rr := httptest.NewRecorder() + // Call the ServeError function + ServeError(rr, tt.err, tt.wantsJSON, tt.data) + + // Check the response + if tt.wantsJSON { + // Check if the response is JSON + var responseMap map[string]interface{} + if err := json.NewDecoder(rr.Body).Decode(&responseMap); err != nil { + t.Fatalf("Failed to decode JSON response: %v", err) + } + + // Check if the expected error is present + if responseMap["error"] != tt.expected { + t.Errorf("Expected error %v, got %v", tt.expected, responseMap["error"]) + } + } else { + // Check if the response body matches the expected error message + if rr.Body.String() != tt.expected { + t.Errorf("Expected response %v, got %v", tt.expected, rr.Body.String()) + } + } + }) + } +} From a56969ad6565d625016e583451316343359e7e0e Mon Sep 17 00:00:00 2001 From: "o.nihilist" <107046146+onihilist@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:38:04 +0100 Subject: [PATCH 003/122] Update server_test.go --- pkg/server/server_test.go | 88 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 25fe7024..4f72dc08 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/gochan-org/gochan/pkg/config" + "github.com/stretchr/testify/assert" ) func TestServeJSON(t *testing.T) { @@ -53,3 +54,90 @@ func TestServeJSON(t *testing.T) { } } } + +func TestServeErrorPage(t *testing.T) { + // Set up a mock configuration + config.SetMockConfig() + + // Set writer and error string message + writer := httptest.NewRecorder() + err := "Unexpected error has occurred." + + ServeErrorPage(writer, err) + + body := writer.Body.String() + t.Log("=============") + t.Log("Response Body:", body) + t.Log("=============") + + // Check response code & content-type + assert.Equal(t, http.StatusOK, writer.Code) + assert.Equal(t, "text/html; charset=utf-8", writer.Header().Get("Content-Type")) + + // Check the response body for the error message + //assert.Contains(t, body, err) // Check if the body contains the error message + //assert.Contains(t, body, "Error") // Check if the body contains the error title or header +} + +func TestServeError(t *testing.T) { + // Set up a mock configuration + config.SetMockConfig() + + tests := []struct { + name string + err string + wantsJSON bool + data map[string]interface{} + expected string + }{ + { + name: "JSON response with error", + err: "some error occurred", + wantsJSON: true, + data: nil, + expected: `some error occurred`, + }, + { + name: "JSON response with existing data", + err: "another error occurred", + wantsJSON: true, + data: map[string]interface{}{"info": "some info"}, + expected: `another error occurred`, + }, + { + name: "Non-JSON response", + err: "page not found", + wantsJSON: false, + data: nil, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a response recorder to capture the response + rr := httptest.NewRecorder() + // Call the ServeError function + ServeError(rr, tt.err, tt.wantsJSON, tt.data) + + // Check the response + if tt.wantsJSON { + // Check if the response is JSON + var responseMap map[string]interface{} + if err := json.NewDecoder(rr.Body).Decode(&responseMap); err != nil { + t.Fatalf("Failed to decode JSON response: %v", err) + } + + // Check if the expected error is present + if responseMap["error"] != tt.expected { + t.Errorf("Expected error %v, got %v", tt.expected, responseMap["error"]) + } + } else { + // Check if the response body matches the expected error message + if rr.Body.String() != tt.expected { + t.Errorf("Expected response %v, got %v", tt.expected, rr.Body.String()) + } + } + }) + } +} From 999e6543b307404256d533fff1502f6990aaa551 Mon Sep 17 00:00:00 2001 From: "o.nihilist" <107046146+onihilist@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:40:40 +0100 Subject: [PATCH 004/122] Add server coverage tests --- pkg/server/server_test.go | 143 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 pkg/server/server_test.go diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go new file mode 100644 index 00000000..4f72dc08 --- /dev/null +++ b/pkg/server/server_test.go @@ -0,0 +1,143 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gochan-org/gochan/pkg/config" + "github.com/stretchr/testify/assert" +) + +func TestServeJSON(t *testing.T) { + // Set up a mock configuration + config.SetMockConfig() + + writer := httptest.NewRecorder() + data := map[string]interface{}{ + "status": "success", + "postID": "777", + "srcBoard": "srcBoard", + "destBoard": "destBoard", + } + + ServeJSON(writer, data) + + // Check the status code + if writer.Code != http.StatusOK { + t.Errorf("expected status code %d, got %d", http.StatusOK, writer.Code) + } + + // Check the content type + if writer.Header().Get("Content-Type") != "application/json" { + t.Errorf("expected content type application/json, got %s", writer.Header().Get("Content-Type")) + } + + // Check the response body + var response map[string]interface{} + if err := json.Unmarshal(writer.Body.Bytes(), &response); err != nil { + t.Fatalf("failed to unmarshal response: %v", err) + } + + // Debugging output + for key, value := range response { + t.Logf("Response key: %s, value: %v", key, value) + } + + for key, expectedValue := range data { + actualValue := response[key] + if actualValue != expectedValue { + t.Errorf("expected %s to be %v, got %v", key, expectedValue, actualValue) + } else { + t.Logf("Match for %s: expected %v, got %v", key, expectedValue, actualValue) + } + } +} + +func TestServeErrorPage(t *testing.T) { + // Set up a mock configuration + config.SetMockConfig() + + // Set writer and error string message + writer := httptest.NewRecorder() + err := "Unexpected error has occurred." + + ServeErrorPage(writer, err) + + body := writer.Body.String() + t.Log("=============") + t.Log("Response Body:", body) + t.Log("=============") + + // Check response code & content-type + assert.Equal(t, http.StatusOK, writer.Code) + assert.Equal(t, "text/html; charset=utf-8", writer.Header().Get("Content-Type")) + + // Check the response body for the error message + //assert.Contains(t, body, err) // Check if the body contains the error message + //assert.Contains(t, body, "Error") // Check if the body contains the error title or header +} + +func TestServeError(t *testing.T) { + // Set up a mock configuration + config.SetMockConfig() + + tests := []struct { + name string + err string + wantsJSON bool + data map[string]interface{} + expected string + }{ + { + name: "JSON response with error", + err: "some error occurred", + wantsJSON: true, + data: nil, + expected: `some error occurred`, + }, + { + name: "JSON response with existing data", + err: "another error occurred", + wantsJSON: true, + data: map[string]interface{}{"info": "some info"}, + expected: `another error occurred`, + }, + { + name: "Non-JSON response", + err: "page not found", + wantsJSON: false, + data: nil, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a response recorder to capture the response + rr := httptest.NewRecorder() + // Call the ServeError function + ServeError(rr, tt.err, tt.wantsJSON, tt.data) + + // Check the response + if tt.wantsJSON { + // Check if the response is JSON + var responseMap map[string]interface{} + if err := json.NewDecoder(rr.Body).Decode(&responseMap); err != nil { + t.Fatalf("Failed to decode JSON response: %v", err) + } + + // Check if the expected error is present + if responseMap["error"] != tt.expected { + t.Errorf("Expected error %v, got %v", tt.expected, responseMap["error"]) + } + } else { + // Check if the response body matches the expected error message + if rr.Body.String() != tt.expected { + t.Errorf("Expected response %v, got %v", tt.expected, rr.Body.String()) + } + } + }) + } +} From c43a15bb0ad13017da11a45a2c61264ab9c72abd Mon Sep 17 00:00:00 2001 From: "o.nihilist" <107046146+onihilist@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:43:16 +0100 Subject: [PATCH 005/122] create SetMockConfig for tests --- pkg/config/config.go | 98 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index e0a17981..89fd4665 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,6 +38,104 @@ type GochanConfig struct { testing bool } +func SetMockConfig() { + cfg = &GochanConfig{ + SystemCriticalConfig: SystemCriticalConfig{ + ListenIP: "127.0.0.1", + Port: 8080, + UseFastCGI: false, + DocumentRoot: "/var/www/html", + TemplateDir: "/var/www/html/templates", + LogDir: "/var/log/gochan", + Plugins: []string{}, + PluginSettings: map[string]any{ + "examplePlugin": true, + }, + SiteHeaderURL: "http://example.com", + WebRoot: "/", + SiteDomain: "example.com", + Verbose: true, + RandomSeed: "testseed", + }, + SiteConfig: SiteConfig{ + FirstPage: []string{"index"}, + Username: "admin", + CookieMaxAge: "1h", + StaffSessionDuration: "2h", + Lockdown: false, + LockdownMessage: "", + SiteName: "Test Site", + SiteSlogan: "A testing site", + Modboard: "mod", + MaxRecentPosts: 10, + RecentPostsWithNoFile: false, + EnableAppeals: true, + MinifyHTML: false, + MinifyJS: false, + GeoIPType: "none", + GeoIPOptions: make(map[string]any), + Captcha: CaptchaConfig{ + Type: "none", + OnlyNeededForThreads: false, + SiteKey: "", + AccountSecret: "", + }, + FingerprintVideoThumbnails: false, + FingerprintHashLength: 16, + }, + BoardConfig: BoardConfig{ + InheritGlobalStyles: true, + Styles: []Style{}, + DefaultStyle: "default", + Banners: []PageBanner{}, + PostConfig: PostConfig{ + MaxLineLength: 1000, + ReservedTrips: []string{}, + ThreadsPerPage: 10, + RepliesOnBoardPage: 5, + StickyRepliesOnBoardPage: 2, + NewThreadsRequireUpload: false, + CyclicalThreadNumPosts: 100, + BanColors: []string{"#FF0000"}, + BanMessage: "You are banned!", + EmbedWidth: 640, + EmbedHeight: 360, + EnableEmbeds: true, + ImagesOpenNewTab: false, + NewTabOnOutlinks: true, + DisableBBcode: false, + }, + UploadConfig: UploadConfig{ + RejectDuplicateImages: true, + ThumbWidth: 150, + ThumbHeight: 150, + ThumbWidthReply: 100, + ThumbHeightReply: 100, + ThumbWidthCatalog: 200, + ThumbHeightCatalog: 200, + AllowOtherExtensions: map[string]string{"pdf": "application/pdf"}, + StripImageMetadata: "none", + ExiftoolPath: "", + }, + DateTimeFormat: "2006-01-02 15:04:05", + ShowPosterID: true, + EnableSpoileredImages: false, + EnableSpoileredThreads: false, + Worksafe: false, + ThreadPage: 1, + Cooldowns: BoardCooldowns{NewThread: 60, Reply: 30, ImageReply: 15}, + RenderURLsAsLinks: true, + ThreadsPerPage: 10, + EnableGeoIP: false, + EnableNoFlag: false, + CustomFlags: []geoip.Country{}, + isGlobal: true, + }, + jsonLocation: "test_config.json", + testing: true, + } +} + // ValidateValues checks to make sure that the configuration options are usable // (e.g., ListenIP is a valid IP address, Port isn't a negative number, etc) func (gcfg *GochanConfig) ValidateValues() error { From 4ae2c564ae16891eeb9be44caabb15d6fe9dc0b3 Mon Sep 17 00:00:00 2001 From: onihilist Date: Thu, 19 Dec 2024 11:06:57 +0100 Subject: [PATCH 006/122] fix code smells for deepsource --- pkg/server/server_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 4f72dc08..9784ed82 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -75,8 +75,8 @@ func TestServeErrorPage(t *testing.T) { assert.Equal(t, "text/html; charset=utf-8", writer.Header().Get("Content-Type")) // Check the response body for the error message - //assert.Contains(t, body, err) // Check if the body contains the error message - //assert.Contains(t, body, "Error") // Check if the body contains the error title or header + //assert.Contains(t, body, err) Check if the body contains the error message + //assert.Contains(t, body, "Error") Check if the body contains the error title or header } func TestServeError(t *testing.T) { @@ -132,11 +132,9 @@ func TestServeError(t *testing.T) { if responseMap["error"] != tt.expected { t.Errorf("Expected error %v, got %v", tt.expected, responseMap["error"]) } - } else { + } else if rr.Body.String() != tt.expected { // Check if the response body matches the expected error message - if rr.Body.String() != tt.expected { - t.Errorf("Expected response %v, got %v", tt.expected, rr.Body.String()) - } + t.Errorf("Expected response %v, got %v", tt.expected, rr.Body.String()) } }) } From 3b8fcc25586b2670be40e840e40044da8d0b6abd Mon Sep 17 00:00:00 2001 From: "o.nihilist" <107046146+onihilist@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:08:02 +0100 Subject: [PATCH 007/122] fix code smells for deepsource --- pkg/server/server_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 4f72dc08..9784ed82 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -75,8 +75,8 @@ func TestServeErrorPage(t *testing.T) { assert.Equal(t, "text/html; charset=utf-8", writer.Header().Get("Content-Type")) // Check the response body for the error message - //assert.Contains(t, body, err) // Check if the body contains the error message - //assert.Contains(t, body, "Error") // Check if the body contains the error title or header + //assert.Contains(t, body, err) Check if the body contains the error message + //assert.Contains(t, body, "Error") Check if the body contains the error title or header } func TestServeError(t *testing.T) { @@ -132,11 +132,9 @@ func TestServeError(t *testing.T) { if responseMap["error"] != tt.expected { t.Errorf("Expected error %v, got %v", tt.expected, responseMap["error"]) } - } else { + } else if rr.Body.String() != tt.expected { // Check if the response body matches the expected error message - if rr.Body.String() != tt.expected { - t.Errorf("Expected response %v, got %v", tt.expected, rr.Body.String()) - } + t.Errorf("Expected response %v, got %v", tt.expected, rr.Body.String()) } }) } From 0b3540a8eda0bead61fec5cd4eb982081d85bb6c Mon Sep 17 00:00:00 2001 From: onihilist Date: Thu, 19 Dec 2024 15:16:11 +0100 Subject: [PATCH 008/122] Add serverstatic_test.go + changed GetSystemCriticalConfig() type return --- pkg/building/building_test.go | 4 +- pkg/config/config.go | 26 ++++---- pkg/config/config_test.go | 16 ++--- pkg/config/preload.go | 4 +- pkg/config/testing.go | 28 ++++---- pkg/config/util.go | 90 +++++++++++++------------- pkg/server/server_test.go | 2 +- pkg/server/serverstatic_test.go | 110 ++++++++++++++++++++++++++++++++ 8 files changed, 195 insertions(+), 85 deletions(-) create mode 100644 pkg/server/serverstatic_test.go diff --git a/pkg/building/building_test.go b/pkg/building/building_test.go index 2b113002..7ff5f5b5 100644 --- a/pkg/building/building_test.go +++ b/pkg/building/building_test.go @@ -32,7 +32,7 @@ func TestBuildJS(t *testing.T) { systemCriticalCfg.LogDir = path.Join(outDir, "logs") systemCriticalCfg.WebRoot = "/chan" systemCriticalCfg.TimeZone = 8 - config.SetSystemCriticalConfig(&systemCriticalCfg) + config.SetSystemCriticalConfig(systemCriticalCfg) boardCfg := config.GetBoardConfig("") boardCfg.Styles = []config.Style{ @@ -154,7 +154,7 @@ func TestBuildFrontPage(t *testing.T) { systemCriticalCfg.LogDir = path.Join(outDir, "logs") systemCriticalCfg.WebRoot = "/chan" systemCriticalCfg.TimeZone = 8 - config.SetSystemCriticalConfig(&systemCriticalCfg) + config.SetSystemCriticalConfig(systemCriticalCfg) boardCfg := config.GetBoardConfig("") boardCfg.Styles = []config.Style{{Name: "test1", Filename: "test1.css"}} diff --git a/pkg/config/config.go b/pkg/config/config.go index 89fd4665..5b3719f2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -24,7 +24,7 @@ const ( ) var ( - cfg *GochanConfig + Cfg *GochanConfig cfgPath string boardConfigs = map[string]BoardConfig{} @@ -39,7 +39,7 @@ type GochanConfig struct { } func SetMockConfig() { - cfg = &GochanConfig{ + Cfg = &GochanConfig{ SystemCriticalConfig: SystemCriticalConfig{ ListenIP: "127.0.0.1", Port: 8080, @@ -432,24 +432,24 @@ type PostConfig struct { } func WriteConfig() error { - return cfg.Write() + return Cfg.Write() } // GetSQLConfig returns SQL configuration info. It returns a value instead of a a pointer to it // because it is not safe to edit while Gochan is running func GetSQLConfig() SQLConfig { - return cfg.SQLConfig + return Cfg.SQLConfig } // GetSystemCriticalConfig returns system-critical configuration options like listening IP // It returns a value instead of a pointer, because it is not usually safe to edit while Gochan is running. -func GetSystemCriticalConfig() SystemCriticalConfig { - return cfg.SystemCriticalConfig +func GetSystemCriticalConfig() *SystemCriticalConfig { + return &Cfg.SystemCriticalConfig } // GetSiteConfig returns the global site configuration (site name, slogan, etc) func GetSiteConfig() *SiteConfig { - return &cfg.SiteConfig + return &Cfg.SiteConfig } // GetBoardConfig returns the custom configuration for the specified board (if it exists) @@ -457,14 +457,14 @@ func GetSiteConfig() *SiteConfig { func GetBoardConfig(board string) *BoardConfig { bc, exists := boardConfigs[board] if board == "" || !exists { - return &cfg.BoardConfig + return &Cfg.BoardConfig } return &bc } // UpdateBoardConfig updates or establishes the configuration for the given board func UpdateBoardConfig(dir string) error { - ba, err := os.ReadFile(path.Join(cfg.DocumentRoot, dir, "board.json")) + ba, err := os.ReadFile(path.Join(Cfg.DocumentRoot, dir, "board.json")) if err != nil { if os.IsNotExist(err) { // board doesn't have a custom config, use global config @@ -472,7 +472,7 @@ func UpdateBoardConfig(dir string) error { } return err } - boardcfg := cfg.BoardConfig + boardcfg := Cfg.BoardConfig if err = json.Unmarshal(ba, &boardcfg); err != nil { return err } @@ -488,13 +488,13 @@ func DeleteBoardConfig(dir string) { } func VerboseMode() bool { - return cfg.testing || cfg.SystemCriticalConfig.Verbose + return Cfg.testing || Cfg.SystemCriticalConfig.Verbose } func SetVerbose(verbose bool) { - cfg.Verbose = verbose + Cfg.Verbose = verbose } func GetVersion() *GochanVersion { - return cfg.Version + return Cfg.Version } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 6d448314..e1066080 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -26,17 +26,17 @@ func TestValidJSON(t *testing.T) { func TestValidateValues(t *testing.T) { InitConfig("3.1.0") SetRandomSeed("test") - assert.NoError(t, cfg.ValidateValues()) + assert.NoError(t, Cfg.ValidateValues()) - cfg.CookieMaxAge = "not a duration" - assert.Error(t, cfg.ValidateValues()) - cfg.CookieMaxAge = "1y" - assert.NoError(t, cfg.ValidateValues()) + Cfg.CookieMaxAge = "not a duration" + assert.Error(t, Cfg.ValidateValues()) + Cfg.CookieMaxAge = "1y" + assert.NoError(t, Cfg.ValidateValues()) SetTestDBConfig("not a valid driver", "127.0.0.1", "gochan", "gochan", "", "") - assert.Error(t, cfg.ValidateValues()) + assert.Error(t, Cfg.ValidateValues()) SetTestDBConfig("postgresql", "127.0.0.1", "gochan", "gochan", "", "") - assert.NoError(t, cfg.ValidateValues()) + assert.NoError(t, Cfg.ValidateValues()) } type webRootTest struct { @@ -61,7 +61,7 @@ func TestWebPath(t *testing.T) { } for _, tC := range testCases { t.Run(tC.expectPath, func(t *testing.T) { - cfg.WebRoot = tC.webRoot + Cfg.WebRoot = tC.webRoot wp := WebPath(tC.pathArgs...) assert.Equal(t, tC.expectPath, wp) }) diff --git a/pkg/config/preload.go b/pkg/config/preload.go index 6a26cd8e..de921750 100644 --- a/pkg/config/preload.go +++ b/pkg/config/preload.go @@ -9,11 +9,11 @@ func PreloadModule(l *lua.LState) int { t := l.NewTable() l.SetFuncs(t, map[string]lua.LGFunction{ "system_critical_config": func(l *lua.LState) int { - l.Push(luar.New(l, &cfg.SystemCriticalConfig)) + l.Push(luar.New(l, &Cfg.SystemCriticalConfig)) return 1 }, "site_config": func(l *lua.LState) int { - l.Push(luar.New(l, &cfg.SiteConfig)) + l.Push(luar.New(l, &Cfg.SiteConfig)) return 1 }, "board_config": func(l *lua.LState) int { diff --git a/pkg/config/testing.go b/pkg/config/testing.go index b0bbec26..c938fa23 100644 --- a/pkg/config/testing.go +++ b/pkg/config/testing.go @@ -3,8 +3,8 @@ package config import "github.com/gochan-org/gochan/pkg/gcutil/testutil" func setDefaultCfgIfNotSet() { - if cfg == nil { - cfg = defaultGochanConfig + if Cfg == nil { + Cfg = defaultGochanConfig } } @@ -13,7 +13,7 @@ func SetVersion(version string) { testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - cfg.Version = ParseVersion(version) + Cfg.Version = ParseVersion(version) } // SetTestTemplateDir sets the directory for templates, used only in testing. If it is not run via `go test`, it will panic. @@ -21,7 +21,7 @@ func SetTestTemplateDir(dir string) { testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - cfg.TemplateDir = dir + Cfg.TemplateDir = dir } // SetTestDBConfig sets up the database configuration for a testing environment. If it is not run via `go test`, it will panic @@ -29,19 +29,19 @@ func SetTestDBConfig(dbType string, dbHost string, dbName string, dbUsername str testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - cfg.DBtype = dbType - cfg.DBhost = dbHost - cfg.DBname = dbName - cfg.DBusername = dbUsername - cfg.DBpassword = dbPassword - cfg.DBprefix = dbPrefix + Cfg.DBtype = dbType + Cfg.DBhost = dbHost + Cfg.DBname = dbName + Cfg.DBusername = dbUsername + Cfg.DBpassword = dbPassword + Cfg.DBprefix = dbPrefix } // SetRandomSeed is usd to set a deterministic seed to make testing easier. If it is not run via `go test`, it will panic func SetRandomSeed(seed string) { testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - cfg.RandomSeed = seed + Cfg.RandomSeed = seed } // SetSystemCriticalConfig sets system critical configuration values in testing. It will panic if it is not run in a @@ -49,14 +49,14 @@ func SetRandomSeed(seed string) { func SetSystemCriticalConfig(systemCritical *SystemCriticalConfig) { testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - cfg.SystemCriticalConfig = *systemCritical + Cfg.SystemCriticalConfig = *systemCritical } // SetSiteConfig sets the site configuration values in testing. It will panic if it is not run in a test environment func SetSiteConfig(siteConfig *SiteConfig) { testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - cfg.SiteConfig = *siteConfig + Cfg.SiteConfig = *siteConfig } // SetBoardConfig applies the configuration to the given board. It will panic if it is not run in a test environment @@ -65,7 +65,7 @@ func SetBoardConfig(board string, boardCfg *BoardConfig) { setDefaultCfgIfNotSet() if board == "" { - cfg.BoardConfig = *boardCfg + Cfg.BoardConfig = *boardCfg } else { boardConfigs[board] = *boardCfg } diff --git a/pkg/config/util.go b/pkg/config/util.go index 30363cea..5ef84745 100644 --- a/pkg/config/util.go +++ b/pkg/config/util.go @@ -54,7 +54,7 @@ func GetUser() (int, int) { } func TakeOwnership(fp string) (err error) { - if runtime.GOOS == "windows" || fp == "" || cfg.Username == "" { + if runtime.GOOS == "windows" || fp == "" || Cfg.Username == "" { // Chown returns an error in Windows so skip it, also skip if Username isn't set // because otherwise it'll think we want to switch to uid and gid 0 (root) return nil @@ -63,7 +63,7 @@ func TakeOwnership(fp string) (err error) { } func TakeOwnershipOfFile(f *os.File) error { - if runtime.GOOS == "windows" || f == nil || cfg.Username == "" { + if runtime.GOOS == "windows" || f == nil || Cfg.Username == "" { // Chown returns an error in Windows so skip it, also skip if Username isn't set // because otherwise it'll think we want to switch to uid and gid 0 (root) return nil @@ -73,27 +73,27 @@ func TakeOwnershipOfFile(f *os.File) error { // InitConfig loads and parses gochan.json on startup and verifies its contents func InitConfig(versionStr string) { - cfg = defaultGochanConfig + Cfg = defaultGochanConfig if strings.HasSuffix(os.Args[0], ".test") { // create a dummy config for testing if we're using go test - cfg = defaultGochanConfig - cfg.ListenIP = "127.0.0.1" - cfg.Port = 8080 - cfg.UseFastCGI = true - cfg.testing = true - cfg.TemplateDir = "templates" - cfg.DBtype = "sqlite3" - cfg.DBhost = "./testdata/gochantest.db" - cfg.DBname = "gochan" - cfg.DBusername = "gochan" - cfg.SiteDomain = "127.0.0.1" - cfg.RandomSeed = "test" - cfg.Version = ParseVersion(versionStr) - cfg.SiteSlogan = "Gochan testing" - cfg.Verbose = true - cfg.Captcha.OnlyNeededForThreads = true - cfg.Cooldowns = BoardCooldowns{0, 0, 0} - cfg.BanColors = []string{ + Cfg = defaultGochanConfig + Cfg.ListenIP = "127.0.0.1" + Cfg.Port = 8080 + Cfg.UseFastCGI = true + Cfg.testing = true + Cfg.TemplateDir = "templates" + Cfg.DBtype = "sqlite3" + Cfg.DBhost = "./testdata/gochantest.db" + Cfg.DBname = "gochan" + Cfg.DBusername = "gochan" + Cfg.SiteDomain = "127.0.0.1" + Cfg.RandomSeed = "test" + Cfg.Version = ParseVersion(versionStr) + Cfg.SiteSlogan = "Gochan testing" + Cfg.Verbose = true + Cfg.Captcha.OnlyNeededForThreads = true + Cfg.Cooldowns = BoardCooldowns{0, 0, 0} + Cfg.BanColors = []string{ "admin:#0000A0", "somemod:blue", } @@ -114,21 +114,21 @@ func InitConfig(versionStr string) { os.Exit(1) } - if err = json.Unmarshal(cfgBytes, cfg); err != nil { + if err = json.Unmarshal(cfgBytes, Cfg); err != nil { fmt.Printf("Error parsing %s: %s", cfgPath, err.Error()) os.Exit(1) } - cfg.jsonLocation = cfgPath + Cfg.jsonLocation = cfgPath - if err = cfg.ValidateValues(); err != nil { + if err = Cfg.ValidateValues(); err != nil { fmt.Println(err.Error()) os.Exit(1) } if runtime.GOOS != "windows" { var gcUser *user.User - if cfg.Username != "" { - gcUser, err = user.Lookup(cfg.Username) + if Cfg.Username != "" { + gcUser, err = user.Lookup(Cfg.Username) } else { gcUser, err = user.Current() } @@ -147,51 +147,51 @@ func InitConfig(versionStr string) { } } - if _, err = os.Stat(cfg.DocumentRoot); err != nil { + if _, err = os.Stat(Cfg.DocumentRoot); err != nil { fmt.Println(err.Error()) os.Exit(1) } - if _, err = os.Stat(cfg.TemplateDir); err != nil { + if _, err = os.Stat(Cfg.TemplateDir); err != nil { fmt.Println(err.Error()) os.Exit(1) } - if _, err = os.Stat(cfg.LogDir); os.IsNotExist(err) { - err = os.MkdirAll(cfg.LogDir, DirFileMode) + if _, err = os.Stat(Cfg.LogDir); os.IsNotExist(err) { + err = os.MkdirAll(Cfg.LogDir, DirFileMode) } if err != nil { fmt.Println(err.Error()) os.Exit(1) } - cfg.LogDir = gcutil.FindResource(cfg.LogDir, "log", "/var/log/gochan/") + Cfg.LogDir = gcutil.FindResource(Cfg.LogDir, "log", "/var/log/gochan/") - if cfg.Port == 0 { - cfg.Port = 80 + if Cfg.Port == 0 { + Cfg.Port = 80 } - if len(cfg.FirstPage) == 0 { - cfg.FirstPage = []string{"index.html", "1.html", "firstrun.html"} + if len(Cfg.FirstPage) == 0 { + Cfg.FirstPage = []string{"index.html", "1.html", "firstrun.html"} } - if cfg.WebRoot == "" { - cfg.WebRoot = "/" + if Cfg.WebRoot == "" { + Cfg.WebRoot = "/" } - if cfg.WebRoot[0] != '/' { - cfg.WebRoot = "/" + cfg.WebRoot + if Cfg.WebRoot[0] != '/' { + Cfg.WebRoot = "/" + Cfg.WebRoot } - if cfg.WebRoot[len(cfg.WebRoot)-1] != '/' { - cfg.WebRoot += "/" + if Cfg.WebRoot[len(Cfg.WebRoot)-1] != '/' { + Cfg.WebRoot += "/" } _, zoneOffset := time.Now().Zone() - cfg.TimeZone = zoneOffset / 60 / 60 + Cfg.TimeZone = zoneOffset / 60 / 60 - cfg.Version = ParseVersion(versionStr) - cfg.Version.Normalize() + Cfg.Version = ParseVersion(versionStr) + Cfg.Version.Normalize() } // WebPath returns an absolute path, starting at the web root (which is "/" by default) func WebPath(part ...string) string { - return path.Join(cfg.WebRoot, path.Join(part...)) + return path.Join(Cfg.WebRoot, path.Join(part...)) } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 9784ed82..09fac76d 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -109,7 +109,7 @@ func TestServeError(t *testing.T) { err: "page not found", wantsJSON: false, data: nil, - expected: "", + expected: "", // Should we expect something ? }, } diff --git a/pkg/server/serverstatic_test.go b/pkg/server/serverstatic_test.go new file mode 100644 index 00000000..0a379d2e --- /dev/null +++ b/pkg/server/serverstatic_test.go @@ -0,0 +1,110 @@ +package server + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path" + "testing" + + "github.com/gochan-org/gochan/pkg/config" + "github.com/stretchr/testify/assert" +) + +func TestServeFile(t *testing.T) { + // Set up a mock configuration + config.SetMockConfig() + + tempDir, err := ioutil.TempDir("", "testservefile") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + // Set the DocumentRoot in the mock config + sysConfig := config.GetSystemCriticalConfig() + sysConfig.DocumentRoot = tempDir + sysConfig.WebRoot = "/" + + // Create a test file + testFileName := "testfile.txt" + testFilePath := path.Join(tempDir, testFileName) + err = ioutil.WriteFile(testFilePath, []byte("Hello, World!"), 0644) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest("GET", "/"+testFileName, nil) + rr := httptest.NewRecorder() + + serveFile(rr, req) + + res := rr.Result() + assert.Equal(t, http.StatusOK, res.StatusCode) + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "Hello, World!", string(body)) +} + +func TestServeFile_NotFound(t *testing.T) { + // Set up a mock configuration + config.SetMockConfig() + + // Create a temporary directory for testing + tempDir, err := ioutil.TempDir("", "testservefile") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) // Clean up after the test + + // Set the DocumentRoot in the mock config + sysConfig := config.GetSystemCriticalConfig() // This should return a pointer now + sysConfig.DocumentRoot = tempDir + sysConfig.WebRoot = "/" + + // Create a request for a non-existent file + req := httptest.NewRequest("GET", "/nonexistentfile.txt", nil) + rr := httptest.NewRecorder() + + // Call the serveFile function + serveFile(rr, req) + + // Check the response + res := rr.Result() + assert.Equal(t, http.StatusNotFound, res.StatusCode) +} + +func TestSetFileHeaders(t *testing.T) { + tests := []struct { + filename string + expectedType string + expectedCache string + }{ + {"image.png", "image/png", "max-age=86400"}, + {"image.gif", "image/gif", "max-age=86400"}, + {"image.jpg", "image/jpeg", "max-age=86400"}, + {"image.jpeg", "image/jpeg", "max-age=86400"}, + {"style.css", "text/css", "max-age=43200"}, + {"script.js", "text/javascript", "max-age=43200"}, + {"data.json", "application/json", "max-age=5, must-revalidate"}, + {"video.webm", "video/webm", "max-age=86400"}, + {"index.html", "text/html", "max-age=5, must-revalidate"}, + {"index.htm", "text/html", "max-age=5, must-revalidate"}, + {"unknownfile.xyz", "application/octet-stream", "max-age=86400"}, + } + + for _, tt := range tests { + t.Run(tt.filename, func(t *testing.T) { + rr := httptest.NewRecorder() + + setFileHeaders(tt.filename, rr) + + assert.Equal(t, tt.expectedType, rr.Header().Get("Content-Type")) + assert.Equal(t, tt.expectedCache, rr.Header().Get("Cache-Control")) + }) + } +} From 358a84c7e30ff7fe6e717055c1330baefa51f0f3 Mon Sep 17 00:00:00 2001 From: onihilist Date: Thu, 19 Dec 2024 15:23:16 +0100 Subject: [PATCH 009/122] Small fix config.go --- pkg/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7a0cb288..f5307d4c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -39,8 +39,8 @@ type GochanConfig struct { } func SetMockConfig() { - - cfg = &GochanConfig{ + + Cfg = &GochanConfig{ SystemCriticalConfig: SystemCriticalConfig{ ListenIP: "127.0.0.1", From 2984f9060babddd3adbfc44f5926179d831ec80e Mon Sep 17 00:00:00 2001 From: "o.nihilist" <107046146+onihilist@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:24:59 +0100 Subject: [PATCH 010/122] small fix config.go --- pkg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7a0cb288..17feed93 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -40,7 +40,7 @@ type GochanConfig struct { func SetMockConfig() { - cfg = &GochanConfig{ + Cfg = &GochanConfig{ SystemCriticalConfig: SystemCriticalConfig{ ListenIP: "127.0.0.1", From 69e40f5cfb479a817b07072f9fef9c436b4fce01 Mon Sep 17 00:00:00 2001 From: onihilist Date: Thu, 19 Dec 2024 15:30:54 +0100 Subject: [PATCH 011/122] Delete launch.json --- pkg/server/.vscode/launch.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 pkg/server/.vscode/launch.json diff --git a/pkg/server/.vscode/launch.json b/pkg/server/.vscode/launch.json deleted file mode 100644 index 5c7247b4..00000000 --- a/pkg/server/.vscode/launch.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [] -} \ No newline at end of file From c6724f0b0685a0b1ae05210f6f885921a25a0b06 Mon Sep 17 00:00:00 2001 From: "o.nihilist" <107046146+onihilist@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:31:38 +0100 Subject: [PATCH 012/122] Delete pkg/server/.vscode directory --- pkg/server/.vscode/launch.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 pkg/server/.vscode/launch.json diff --git a/pkg/server/.vscode/launch.json b/pkg/server/.vscode/launch.json deleted file mode 100644 index 5c7247b4..00000000 --- a/pkg/server/.vscode/launch.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [] -} \ No newline at end of file From 2b0a6a9d54302c84850949775a05947f7ae0f64b Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Fri, 20 Dec 2024 23:06:06 -0800 Subject: [PATCH 013/122] Call InitMinifier in canMinify in case it hasn't been called already elsewhere --- pkg/server/serverutil/minifier.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/server/serverutil/minifier.go b/pkg/server/serverutil/minifier.go index 8570b75a..6e1dd4f1 100644 --- a/pkg/server/serverutil/minifier.go +++ b/pkg/server/serverutil/minifier.go @@ -34,7 +34,12 @@ func InitMinifier() { } } -func canMinify(mediaType string) bool { +func canMinify(mediaType string) (minify bool) { + defer func() { + if minify && minifier == nil { + InitMinifier() + } + }() siteConfig := config.GetSiteConfig() if mediaType == "text/html" && siteConfig.MinifyHTML { return true From 64e1e6e15afe457455bfd2efbd88dd5f85fabb5e Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Mon, 23 Dec 2024 09:19:05 -0800 Subject: [PATCH 014/122] Fix PanicIfNotTest and GoToGochanRoot --- pkg/gcutil/testutil/testutil.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/gcutil/testutil/testutil.go b/pkg/gcutil/testutil/testutil.go index 37755a37..b2fb5974 100644 --- a/pkg/gcutil/testutil/testutil.go +++ b/pkg/gcutil/testutil/testutil.go @@ -3,7 +3,7 @@ package testutil import ( "errors" "os" - "path" + "path/filepath" "strings" "testing" ) @@ -14,7 +14,7 @@ const ( // PanicIfNotTest panics if the function was called directly or indirectly by a test function via go test func PanicIfNotTest() { - if !strings.HasSuffix(os.Args[0], ".test") && os.Args[1] != "-test.run" { + if !strings.HasSuffix(os.Args[0], ".test") && !strings.HasSuffix(os.Args[0], ".test.exe") && os.Args[1] != "-test.run" { panic("the testutil package should only be used in tests") } } @@ -30,7 +30,7 @@ func GoToGochanRoot(t *testing.T) (string, error) { if err != nil { return "", err } - if path.Base(dir) == "gochan" { + if filepath.Base(dir) == "gochan" { return dir, nil } if err = os.Chdir(".."); err != nil { From 3794a44a101873bc282d2deae105b54a94fc674d Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Mon, 23 Dec 2024 16:47:46 -0800 Subject: [PATCH 015/122] Update expected results for consts.js tests --- pkg/gctemplates/templatetests/templatecases_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/gctemplates/templatetests/templatecases_test.go b/pkg/gctemplates/templatetests/templatecases_test.go index a56681b4..affaf46a 100644 --- a/pkg/gctemplates/templatetests/templatecases_test.go +++ b/pkg/gctemplates/templatetests/templatecases_test.go @@ -230,7 +230,7 @@ var ( ".ext": "thumb.png", }, }, - expectedOutput: `var styles=[{Name:"Pipes",Filename:"pipes.css"},{Name:"Yotsuba A",Filename:"yotsuba.css"}];var defaultStyle="pipes.css";var webroot="/";var serverTZ=-1;var fileTypes=[".ext",];`, + expectedOutput: `const styles=[{Name:"Pipes",Filename:"pipes.css"},{Name:"Yotsuba A",Filename:"yotsuba.css"}];const defaultStyle="pipes.css";const webroot="/";const serverTZ=-1;const fileTypes=[".ext",];`, }, { desc: "empty values", @@ -239,7 +239,7 @@ var ( "webroot": "", "timezone": 0, }, - expectedOutput: `var styles=[];var defaultStyle="";var webroot="";var serverTZ=0;var fileTypes=[];`, + expectedOutput: `const styles=[];const defaultStyle="";const webroot="";const serverTZ=0;const fileTypes=[];`, }, { desc: "escaped string", @@ -248,7 +248,7 @@ var ( "webroot": "", "timezone": 0, }, - expectedOutput: `var styles=[];var defaultStyle="\"a\\a\"";var webroot="";var serverTZ=0;var fileTypes=[];`, + expectedOutput: `const styles=[];const defaultStyle="\"a\\a\"";const webroot="";const serverTZ=0;const fileTypes=[];`, }, } ) From 9c538c8d10a3b30884103b2052925c8d366f1cb2 Mon Sep 17 00:00:00 2001 From: onihilist Date: Tue, 24 Dec 2024 08:23:25 +0100 Subject: [PATCH 016/122] Merge testutil fix & fix tests --- pkg/config/config.go | 122 +++----------------------------- pkg/config/config_test.go | 16 ++--- pkg/config/preload.go | 4 +- pkg/config/testing.go | 28 ++++---- pkg/config/util.go | 90 +++++++++++------------ pkg/server/server_test.go | 37 +++++++--- pkg/server/serverstatic_test.go | 8 +-- 7 files changed, 111 insertions(+), 194 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index f5307d4c..650dc544 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -24,7 +24,7 @@ const ( ) var ( - Cfg *GochanConfig + cfg *GochanConfig cfgPath string boardConfigs = map[string]BoardConfig{} @@ -38,106 +38,6 @@ type GochanConfig struct { testing bool } -func SetMockConfig() { - - Cfg = &GochanConfig{ - - SystemCriticalConfig: SystemCriticalConfig{ - ListenIP: "127.0.0.1", - Port: 8080, - UseFastCGI: false, - DocumentRoot: "/var/www/html", - TemplateDir: "/var/www/html/templates", - LogDir: "/var/log/gochan", - Plugins: []string{}, - PluginSettings: map[string]any{ - "examplePlugin": true, - }, - SiteHeaderURL: "http://example.com", - WebRoot: "/", - SiteDomain: "example.com", - Verbose: true, - RandomSeed: "testseed", - }, - SiteConfig: SiteConfig{ - FirstPage: []string{"index"}, - Username: "admin", - CookieMaxAge: "1h", - StaffSessionDuration: "2h", - Lockdown: false, - LockdownMessage: "", - SiteName: "Test Site", - SiteSlogan: "A testing site", - Modboard: "mod", - MaxRecentPosts: 10, - RecentPostsWithNoFile: false, - EnableAppeals: true, - MinifyHTML: false, - MinifyJS: false, - GeoIPType: "none", - GeoIPOptions: make(map[string]any), - Captcha: CaptchaConfig{ - Type: "none", - OnlyNeededForThreads: false, - SiteKey: "", - AccountSecret: "", - }, - FingerprintVideoThumbnails: false, - FingerprintHashLength: 16, - }, - BoardConfig: BoardConfig{ - InheritGlobalStyles: true, - Styles: []Style{}, - DefaultStyle: "default", - Banners: []PageBanner{}, - PostConfig: PostConfig{ - MaxLineLength: 1000, - ReservedTrips: []string{}, - ThreadsPerPage: 10, - RepliesOnBoardPage: 5, - StickyRepliesOnBoardPage: 2, - NewThreadsRequireUpload: false, - CyclicalThreadNumPosts: 100, - BanColors: []string{"#FF0000"}, - BanMessage: "You are banned!", - EmbedWidth: 640, - EmbedHeight: 360, - EnableEmbeds: true, - ImagesOpenNewTab: false, - NewTabOnOutlinks: true, - DisableBBcode: false, - }, - UploadConfig: UploadConfig{ - RejectDuplicateImages: true, - ThumbWidth: 150, - ThumbHeight: 150, - ThumbWidthReply: 100, - ThumbHeightReply: 100, - ThumbWidthCatalog: 200, - ThumbHeightCatalog: 200, - AllowOtherExtensions: map[string]string{"pdf": "application/pdf"}, - StripImageMetadata: "none", - ExiftoolPath: "", - }, - DateTimeFormat: "2006-01-02 15:04:05", - ShowPosterID: true, - EnableSpoileredImages: false, - EnableSpoileredThreads: false, - Worksafe: false, - ThreadPage: 1, - Cooldowns: BoardCooldowns{NewThread: 60, Reply: 30, ImageReply: 15}, - RenderURLsAsLinks: true, - ThreadsPerPage: 10, - EnableGeoIP: false, - EnableNoFlag: false, - CustomFlags: []geoip.Country{}, - isGlobal: true, - }, - jsonLocation: "test_config.json", - testing: true, - } -} - // ValidateValues checks to make sure that the configuration options are usable // (e.g., ListenIP is a valid IP address, Port isn't a negative number, etc) func (gcfg *GochanConfig) ValidateValues() error { @@ -434,24 +334,24 @@ type PostConfig struct { } func WriteConfig() error { - return Cfg.Write() + return cfg.Write() } // GetSQLConfig returns SQL configuration info. It returns a value instead of a a pointer to it // because it is not safe to edit while Gochan is running func GetSQLConfig() SQLConfig { - return Cfg.SQLConfig + return cfg.SQLConfig } // GetSystemCriticalConfig returns system-critical configuration options like listening IP // It returns a value instead of a pointer, because it is not usually safe to edit while Gochan is running. func GetSystemCriticalConfig() *SystemCriticalConfig { - return &Cfg.SystemCriticalConfig + return &cfg.SystemCriticalConfig } // GetSiteConfig returns the global site configuration (site name, slogan, etc) func GetSiteConfig() *SiteConfig { - return &Cfg.SiteConfig + return &cfg.SiteConfig } // GetBoardConfig returns the custom configuration for the specified board (if it exists) @@ -459,14 +359,14 @@ func GetSiteConfig() *SiteConfig { func GetBoardConfig(board string) *BoardConfig { bc, exists := boardConfigs[board] if board == "" || !exists { - return &Cfg.BoardConfig + return &cfg.BoardConfig } return &bc } // UpdateBoardConfig updates or establishes the configuration for the given board func UpdateBoardConfig(dir string) error { - ba, err := os.ReadFile(path.Join(Cfg.DocumentRoot, dir, "board.json")) + ba, err := os.ReadFile(path.Join(cfg.DocumentRoot, dir, "board.json")) if err != nil { if os.IsNotExist(err) { // board doesn't have a custom config, use global config @@ -474,7 +374,7 @@ func UpdateBoardConfig(dir string) error { } return err } - boardcfg := Cfg.BoardConfig + boardcfg := cfg.BoardConfig if err = json.Unmarshal(ba, &boardcfg); err != nil { return err } @@ -490,13 +390,13 @@ func DeleteBoardConfig(dir string) { } func VerboseMode() bool { - return Cfg.testing || Cfg.SystemCriticalConfig.Verbose + return cfg.testing || cfg.SystemCriticalConfig.Verbose } func SetVerbose(verbose bool) { - Cfg.Verbose = verbose + cfg.Verbose = verbose } func GetVersion() *GochanVersion { - return Cfg.Version + return cfg.Version } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index e1066080..6d448314 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -26,17 +26,17 @@ func TestValidJSON(t *testing.T) { func TestValidateValues(t *testing.T) { InitConfig("3.1.0") SetRandomSeed("test") - assert.NoError(t, Cfg.ValidateValues()) + assert.NoError(t, cfg.ValidateValues()) - Cfg.CookieMaxAge = "not a duration" - assert.Error(t, Cfg.ValidateValues()) - Cfg.CookieMaxAge = "1y" - assert.NoError(t, Cfg.ValidateValues()) + cfg.CookieMaxAge = "not a duration" + assert.Error(t, cfg.ValidateValues()) + cfg.CookieMaxAge = "1y" + assert.NoError(t, cfg.ValidateValues()) SetTestDBConfig("not a valid driver", "127.0.0.1", "gochan", "gochan", "", "") - assert.Error(t, Cfg.ValidateValues()) + assert.Error(t, cfg.ValidateValues()) SetTestDBConfig("postgresql", "127.0.0.1", "gochan", "gochan", "", "") - assert.NoError(t, Cfg.ValidateValues()) + assert.NoError(t, cfg.ValidateValues()) } type webRootTest struct { @@ -61,7 +61,7 @@ func TestWebPath(t *testing.T) { } for _, tC := range testCases { t.Run(tC.expectPath, func(t *testing.T) { - Cfg.WebRoot = tC.webRoot + cfg.WebRoot = tC.webRoot wp := WebPath(tC.pathArgs...) assert.Equal(t, tC.expectPath, wp) }) diff --git a/pkg/config/preload.go b/pkg/config/preload.go index de921750..6a26cd8e 100644 --- a/pkg/config/preload.go +++ b/pkg/config/preload.go @@ -9,11 +9,11 @@ func PreloadModule(l *lua.LState) int { t := l.NewTable() l.SetFuncs(t, map[string]lua.LGFunction{ "system_critical_config": func(l *lua.LState) int { - l.Push(luar.New(l, &Cfg.SystemCriticalConfig)) + l.Push(luar.New(l, &cfg.SystemCriticalConfig)) return 1 }, "site_config": func(l *lua.LState) int { - l.Push(luar.New(l, &Cfg.SiteConfig)) + l.Push(luar.New(l, &cfg.SiteConfig)) return 1 }, "board_config": func(l *lua.LState) int { diff --git a/pkg/config/testing.go b/pkg/config/testing.go index c938fa23..b0bbec26 100644 --- a/pkg/config/testing.go +++ b/pkg/config/testing.go @@ -3,8 +3,8 @@ package config import "github.com/gochan-org/gochan/pkg/gcutil/testutil" func setDefaultCfgIfNotSet() { - if Cfg == nil { - Cfg = defaultGochanConfig + if cfg == nil { + cfg = defaultGochanConfig } } @@ -13,7 +13,7 @@ func SetVersion(version string) { testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - Cfg.Version = ParseVersion(version) + cfg.Version = ParseVersion(version) } // SetTestTemplateDir sets the directory for templates, used only in testing. If it is not run via `go test`, it will panic. @@ -21,7 +21,7 @@ func SetTestTemplateDir(dir string) { testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - Cfg.TemplateDir = dir + cfg.TemplateDir = dir } // SetTestDBConfig sets up the database configuration for a testing environment. If it is not run via `go test`, it will panic @@ -29,19 +29,19 @@ func SetTestDBConfig(dbType string, dbHost string, dbName string, dbUsername str testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - Cfg.DBtype = dbType - Cfg.DBhost = dbHost - Cfg.DBname = dbName - Cfg.DBusername = dbUsername - Cfg.DBpassword = dbPassword - Cfg.DBprefix = dbPrefix + cfg.DBtype = dbType + cfg.DBhost = dbHost + cfg.DBname = dbName + cfg.DBusername = dbUsername + cfg.DBpassword = dbPassword + cfg.DBprefix = dbPrefix } // SetRandomSeed is usd to set a deterministic seed to make testing easier. If it is not run via `go test`, it will panic func SetRandomSeed(seed string) { testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - Cfg.RandomSeed = seed + cfg.RandomSeed = seed } // SetSystemCriticalConfig sets system critical configuration values in testing. It will panic if it is not run in a @@ -49,14 +49,14 @@ func SetRandomSeed(seed string) { func SetSystemCriticalConfig(systemCritical *SystemCriticalConfig) { testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - Cfg.SystemCriticalConfig = *systemCritical + cfg.SystemCriticalConfig = *systemCritical } // SetSiteConfig sets the site configuration values in testing. It will panic if it is not run in a test environment func SetSiteConfig(siteConfig *SiteConfig) { testutil.PanicIfNotTest() setDefaultCfgIfNotSet() - Cfg.SiteConfig = *siteConfig + cfg.SiteConfig = *siteConfig } // SetBoardConfig applies the configuration to the given board. It will panic if it is not run in a test environment @@ -65,7 +65,7 @@ func SetBoardConfig(board string, boardCfg *BoardConfig) { setDefaultCfgIfNotSet() if board == "" { - Cfg.BoardConfig = *boardCfg + cfg.BoardConfig = *boardCfg } else { boardConfigs[board] = *boardCfg } diff --git a/pkg/config/util.go b/pkg/config/util.go index 5ef84745..30363cea 100644 --- a/pkg/config/util.go +++ b/pkg/config/util.go @@ -54,7 +54,7 @@ func GetUser() (int, int) { } func TakeOwnership(fp string) (err error) { - if runtime.GOOS == "windows" || fp == "" || Cfg.Username == "" { + if runtime.GOOS == "windows" || fp == "" || cfg.Username == "" { // Chown returns an error in Windows so skip it, also skip if Username isn't set // because otherwise it'll think we want to switch to uid and gid 0 (root) return nil @@ -63,7 +63,7 @@ func TakeOwnership(fp string) (err error) { } func TakeOwnershipOfFile(f *os.File) error { - if runtime.GOOS == "windows" || f == nil || Cfg.Username == "" { + if runtime.GOOS == "windows" || f == nil || cfg.Username == "" { // Chown returns an error in Windows so skip it, also skip if Username isn't set // because otherwise it'll think we want to switch to uid and gid 0 (root) return nil @@ -73,27 +73,27 @@ func TakeOwnershipOfFile(f *os.File) error { // InitConfig loads and parses gochan.json on startup and verifies its contents func InitConfig(versionStr string) { - Cfg = defaultGochanConfig + cfg = defaultGochanConfig if strings.HasSuffix(os.Args[0], ".test") { // create a dummy config for testing if we're using go test - Cfg = defaultGochanConfig - Cfg.ListenIP = "127.0.0.1" - Cfg.Port = 8080 - Cfg.UseFastCGI = true - Cfg.testing = true - Cfg.TemplateDir = "templates" - Cfg.DBtype = "sqlite3" - Cfg.DBhost = "./testdata/gochantest.db" - Cfg.DBname = "gochan" - Cfg.DBusername = "gochan" - Cfg.SiteDomain = "127.0.0.1" - Cfg.RandomSeed = "test" - Cfg.Version = ParseVersion(versionStr) - Cfg.SiteSlogan = "Gochan testing" - Cfg.Verbose = true - Cfg.Captcha.OnlyNeededForThreads = true - Cfg.Cooldowns = BoardCooldowns{0, 0, 0} - Cfg.BanColors = []string{ + cfg = defaultGochanConfig + cfg.ListenIP = "127.0.0.1" + cfg.Port = 8080 + cfg.UseFastCGI = true + cfg.testing = true + cfg.TemplateDir = "templates" + cfg.DBtype = "sqlite3" + cfg.DBhost = "./testdata/gochantest.db" + cfg.DBname = "gochan" + cfg.DBusername = "gochan" + cfg.SiteDomain = "127.0.0.1" + cfg.RandomSeed = "test" + cfg.Version = ParseVersion(versionStr) + cfg.SiteSlogan = "Gochan testing" + cfg.Verbose = true + cfg.Captcha.OnlyNeededForThreads = true + cfg.Cooldowns = BoardCooldowns{0, 0, 0} + cfg.BanColors = []string{ "admin:#0000A0", "somemod:blue", } @@ -114,21 +114,21 @@ func InitConfig(versionStr string) { os.Exit(1) } - if err = json.Unmarshal(cfgBytes, Cfg); err != nil { + if err = json.Unmarshal(cfgBytes, cfg); err != nil { fmt.Printf("Error parsing %s: %s", cfgPath, err.Error()) os.Exit(1) } - Cfg.jsonLocation = cfgPath + cfg.jsonLocation = cfgPath - if err = Cfg.ValidateValues(); err != nil { + if err = cfg.ValidateValues(); err != nil { fmt.Println(err.Error()) os.Exit(1) } if runtime.GOOS != "windows" { var gcUser *user.User - if Cfg.Username != "" { - gcUser, err = user.Lookup(Cfg.Username) + if cfg.Username != "" { + gcUser, err = user.Lookup(cfg.Username) } else { gcUser, err = user.Current() } @@ -147,51 +147,51 @@ func InitConfig(versionStr string) { } } - if _, err = os.Stat(Cfg.DocumentRoot); err != nil { + if _, err = os.Stat(cfg.DocumentRoot); err != nil { fmt.Println(err.Error()) os.Exit(1) } - if _, err = os.Stat(Cfg.TemplateDir); err != nil { + if _, err = os.Stat(cfg.TemplateDir); err != nil { fmt.Println(err.Error()) os.Exit(1) } - if _, err = os.Stat(Cfg.LogDir); os.IsNotExist(err) { - err = os.MkdirAll(Cfg.LogDir, DirFileMode) + if _, err = os.Stat(cfg.LogDir); os.IsNotExist(err) { + err = os.MkdirAll(cfg.LogDir, DirFileMode) } if err != nil { fmt.Println(err.Error()) os.Exit(1) } - Cfg.LogDir = gcutil.FindResource(Cfg.LogDir, "log", "/var/log/gochan/") + cfg.LogDir = gcutil.FindResource(cfg.LogDir, "log", "/var/log/gochan/") - if Cfg.Port == 0 { - Cfg.Port = 80 + if cfg.Port == 0 { + cfg.Port = 80 } - if len(Cfg.FirstPage) == 0 { - Cfg.FirstPage = []string{"index.html", "1.html", "firstrun.html"} + if len(cfg.FirstPage) == 0 { + cfg.FirstPage = []string{"index.html", "1.html", "firstrun.html"} } - if Cfg.WebRoot == "" { - Cfg.WebRoot = "/" + if cfg.WebRoot == "" { + cfg.WebRoot = "/" } - if Cfg.WebRoot[0] != '/' { - Cfg.WebRoot = "/" + Cfg.WebRoot + if cfg.WebRoot[0] != '/' { + cfg.WebRoot = "/" + cfg.WebRoot } - if Cfg.WebRoot[len(Cfg.WebRoot)-1] != '/' { - Cfg.WebRoot += "/" + if cfg.WebRoot[len(cfg.WebRoot)-1] != '/' { + cfg.WebRoot += "/" } _, zoneOffset := time.Now().Zone() - Cfg.TimeZone = zoneOffset / 60 / 60 + cfg.TimeZone = zoneOffset / 60 / 60 - Cfg.Version = ParseVersion(versionStr) - Cfg.Version.Normalize() + cfg.Version = ParseVersion(versionStr) + cfg.Version.Normalize() } // WebPath returns an absolute path, starting at the web root (which is "/" by default) func WebPath(part ...string) string { - return path.Join(Cfg.WebRoot, path.Join(part...)) + return path.Join(cfg.WebRoot, path.Join(part...)) } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 09fac76d..91f31d2d 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -2,17 +2,27 @@ package server import ( "encoding/json" + "fmt" "net/http" "net/http/httptest" "testing" "github.com/gochan-org/gochan/pkg/config" + "github.com/gochan-org/gochan/pkg/gcutil/testutil" "github.com/stretchr/testify/assert" ) func TestServeJSON(t *testing.T) { - // Set up a mock configuration - config.SetMockConfig() + + dir, err := testutil.GoToGochanRoot(t) + if !assert.NoError(t, err) { + t.Fatalf("Failed to get current working directory: %v", err) + fmt.Println("Current working directory:", dir) + return + } + + config.SetVersion("4.0.1") + config.SetRandomSeed("test") writer := httptest.NewRecorder() data := map[string]interface{}{ @@ -56,14 +66,21 @@ func TestServeJSON(t *testing.T) { } func TestServeErrorPage(t *testing.T) { - // Set up a mock configuration - config.SetMockConfig() + + dir, err := testutil.GoToGochanRoot(t) + if !assert.NoError(t, err) { + t.Fatalf("Failed to get current working directory: %v", err) + fmt.Println("Current working directory:", dir) + return + } + + config.SetVersion("4.0.1") // Set writer and error string message writer := httptest.NewRecorder() - err := "Unexpected error has occurred." + errorMsg := "Unexpected error has occurred." - ServeErrorPage(writer, err) + ServeErrorPage(writer, errorMsg) body := writer.Body.String() t.Log("=============") @@ -75,13 +92,13 @@ func TestServeErrorPage(t *testing.T) { assert.Equal(t, "text/html; charset=utf-8", writer.Header().Get("Content-Type")) // Check the response body for the error message - //assert.Contains(t, body, err) Check if the body contains the error message - //assert.Contains(t, body, "Error") Check if the body contains the error title or header + assert.Contains(t, body, err) + assert.Contains(t, body, "Error") } func TestServeError(t *testing.T) { - // Set up a mock configuration - config.SetMockConfig() + + config.SetVersion("4.0.1") tests := []struct { name string diff --git a/pkg/server/serverstatic_test.go b/pkg/server/serverstatic_test.go index 0a379d2e..11372a6c 100644 --- a/pkg/server/serverstatic_test.go +++ b/pkg/server/serverstatic_test.go @@ -13,8 +13,8 @@ import ( ) func TestServeFile(t *testing.T) { - // Set up a mock configuration - config.SetMockConfig() + + config.SetVersion("4.0.1") tempDir, err := ioutil.TempDir("", "testservefile") if err != nil { @@ -51,8 +51,8 @@ func TestServeFile(t *testing.T) { } func TestServeFile_NotFound(t *testing.T) { - // Set up a mock configuration - config.SetMockConfig() + + config.SetVersion("4.0.1") // Create a temporary directory for testing tempDir, err := ioutil.TempDir("", "testservefile") From cb1bda430bfdad291f409c6465c2459f1f2b37a5 Mon Sep 17 00:00:00 2001 From: onihilist Date: Tue, 24 Dec 2024 08:32:46 +0100 Subject: [PATCH 017/122] deepsource fix --- pkg/server/server_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 91f31d2d..451647ec 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -2,7 +2,6 @@ package server import ( "encoding/json" - "fmt" "net/http" "net/http/httptest" "testing" @@ -14,10 +13,9 @@ import ( func TestServeJSON(t *testing.T) { - dir, err := testutil.GoToGochanRoot(t) + _, err := testutil.GoToGochanRoot(t) if !assert.NoError(t, err) { t.Fatalf("Failed to get current working directory: %v", err) - fmt.Println("Current working directory:", dir) return } @@ -67,10 +65,9 @@ func TestServeJSON(t *testing.T) { func TestServeErrorPage(t *testing.T) { - dir, err := testutil.GoToGochanRoot(t) + _, err := testutil.GoToGochanRoot(t) if !assert.NoError(t, err) { t.Fatalf("Failed to get current working directory: %v", err) - fmt.Println("Current working directory:", dir) return } From 9170c18df61119e82d01a8ae0ca07ae605fab847 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Tue, 24 Dec 2024 23:19:24 -0800 Subject: [PATCH 018/122] Properly attach file to QR submitted form, prevent scrolling on QR submission --- frontend/ts/dom/qr.ts | 16 ++++++++++++++++ frontend/ts/dom/uploaddata.ts | 12 +++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/frontend/ts/dom/qr.ts b/frontend/ts/dom/qr.ts index 25c6f612..47a34b47 100755 --- a/frontend/ts/dom/qr.ts +++ b/frontend/ts/dom/qr.ts @@ -93,6 +93,21 @@ function setButtonTimeout(prefix = "", cooldown = 5) { timeoutCB(); } +function fixFileList() { + let files:FileList = null; + const $browseBtns = $("input[name=imagefile]"); + $browseBtns.each((_i, el) => { + if(el.files?.length > 0 && !files) { + files = el.files; + return false; + } + }); + $browseBtns.each((_i, el) => { + if(files) + el.files = files; + }); +} + export function initQR() { if($qr !== null) { // QR box already initialized @@ -204,6 +219,7 @@ export function initQR() { resetSubmitButtonText(); $postform.on("submit", function(e) { + fixFileList(); const $form = $(this); e.preventDefault(); copyCaptchaResponse($form); diff --git a/frontend/ts/dom/uploaddata.ts b/frontend/ts/dom/uploaddata.ts index 61c7afe3..f82948db 100644 --- a/frontend/ts/dom/uploaddata.ts +++ b/frontend/ts/dom/uploaddata.ts @@ -86,13 +86,19 @@ function replaceBrowseButton() { $("").addClass("browse-text") .attr("href", "#") .text("Select/drop/paste upload here") - .on("click", () => $browseBtn.trigger("click")) + .on("click", e => { + e.preventDefault(); + $browseBtn.trigger("click"); + }) ).on("dragenter dragover drop", dragAndDrop).insertBefore($browseBtn); $("form#postform, form#qrpostform").on("paste", e => { const clipboardData = (e.originalEvent as ClipboardEvent).clipboardData; - if(clipboardData.items.length < 1 || clipboardData.items[0].kind !== "file") { - console.log("No files in clipboard"); + if(clipboardData.items.length < 1) { + alertLightbox("No files in clipboard", "Unable to paste"); + return; + } + if(clipboardData.items[0].kind !== "file") { return; } const clipboardFile = clipboardData.items[0].getAsFile(); From 10be5cb0f3505c80472f739430c2927801b0b130 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 25 Dec 2024 14:20:57 -0800 Subject: [PATCH 019/122] Update golang.org/x/net dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 980e2c0a..684bd028 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/yuin/gopher-lua v1.1.1 golang.org/x/crypto v0.31.0 golang.org/x/image v0.23.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf layeh.com/gopher-luar v1.0.11 ) diff --git a/go.sum b/go.sum index b22a5be0..8310c354 100644 --- a/go.sum +++ b/go.sum @@ -189,8 +189,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 6b263ca48f6eb7b33a35fd56aeebf68b8008babd Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 25 Dec 2024 14:43:57 -0800 Subject: [PATCH 020/122] Replace ioutil with io and os, fix serverstatic_test.go tests --- pkg/server/server_test.go | 16 ++++++++++------ pkg/server/serverstatic_test.go | 10 +++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 451647ec..70f8a2c9 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -7,7 +7,10 @@ import ( "testing" "github.com/gochan-org/gochan/pkg/config" + _ "github.com/gochan-org/gochan/pkg/gcsql/initsql" + "github.com/gochan-org/gochan/pkg/gctemplates" "github.com/gochan-org/gochan/pkg/gcutil/testutil" + _ "github.com/gochan-org/gochan/pkg/posting/uploads/inituploads" "github.com/stretchr/testify/assert" ) @@ -64,14 +67,15 @@ func TestServeJSON(t *testing.T) { } func TestServeErrorPage(t *testing.T) { - _, err := testutil.GoToGochanRoot(t) - if !assert.NoError(t, err) { - t.Fatalf("Failed to get current working directory: %v", err) + if !assert.NoError(t, err, "Unable to switch to gochan root directory") { return } - config.SetVersion("4.0.1") + config.SetTestTemplateDir("templates") + if !assert.NoError(t, gctemplates.InitTemplates()) { + return + } // Set writer and error string message writer := httptest.NewRecorder() @@ -89,7 +93,7 @@ func TestServeErrorPage(t *testing.T) { assert.Equal(t, "text/html; charset=utf-8", writer.Header().Get("Content-Type")) // Check the response body for the error message - assert.Contains(t, body, err) + assert.Contains(t, body, errorMsg) assert.Contains(t, body, "Error") } @@ -123,7 +127,7 @@ func TestServeError(t *testing.T) { err: "page not found", wantsJSON: false, data: nil, - expected: "", // Should we expect something ? + expected: "Error :c

Error

page not found


Site powered by Gochan 4.0.1
", }, } diff --git a/pkg/server/serverstatic_test.go b/pkg/server/serverstatic_test.go index 11372a6c..78b6c68b 100644 --- a/pkg/server/serverstatic_test.go +++ b/pkg/server/serverstatic_test.go @@ -1,7 +1,7 @@ package server import ( - "io/ioutil" + "io" "net/http" "net/http/httptest" "os" @@ -16,7 +16,7 @@ func TestServeFile(t *testing.T) { config.SetVersion("4.0.1") - tempDir, err := ioutil.TempDir("", "testservefile") + tempDir, err := os.MkdirTemp("", "testservefile") if err != nil { t.Fatal(err) } @@ -30,7 +30,7 @@ func TestServeFile(t *testing.T) { // Create a test file testFileName := "testfile.txt" testFilePath := path.Join(tempDir, testFileName) - err = ioutil.WriteFile(testFilePath, []byte("Hello, World!"), 0644) + err = os.WriteFile(testFilePath, []byte("Hello, World!"), 0644) if err != nil { t.Fatal(err) } @@ -43,7 +43,7 @@ func TestServeFile(t *testing.T) { res := rr.Result() assert.Equal(t, http.StatusOK, res.StatusCode) - body, err := ioutil.ReadAll(res.Body) + body, err := io.ReadAll(res.Body) if err != nil { t.Fatal(err) } @@ -55,7 +55,7 @@ func TestServeFile_NotFound(t *testing.T) { config.SetVersion("4.0.1") // Create a temporary directory for testing - tempDir, err := ioutil.TempDir("", "testservefile") + tempDir, err := os.MkdirTemp("", "testservefile") if err != nil { t.Fatal(err) } From 8dd8214974ff95d6c79b05b6089c0fe7f448a017 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 25 Dec 2024 14:54:04 -0800 Subject: [PATCH 021/122] Fix deepsource JS-0045 --- frontend/ts/dom/qr.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/ts/dom/qr.ts b/frontend/ts/dom/qr.ts index 47a34b47..ee84ef09 100755 --- a/frontend/ts/dom/qr.ts +++ b/frontend/ts/dom/qr.ts @@ -101,6 +101,7 @@ function fixFileList() { files = el.files; return false; } + return; }); $browseBtns.each((_i, el) => { if(files) From 3bec5d2c3d69220a16173ea7b19c89def17b53a2 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 25 Dec 2024 14:57:52 -0800 Subject: [PATCH 022/122] Fix JS-0045 really this time --- frontend/ts/dom/qr.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/ts/dom/qr.ts b/frontend/ts/dom/qr.ts index ee84ef09..3aa5d549 100755 --- a/frontend/ts/dom/qr.ts +++ b/frontend/ts/dom/qr.ts @@ -101,7 +101,7 @@ function fixFileList() { files = el.files; return false; } - return; + return null; }); $browseBtns.each((_i, el) => { if(files) From 89c7a64da0c0168a230af604c936373a3652caa7 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 29 Dec 2024 16:54:36 -0800 Subject: [PATCH 023/122] Don't require screen in get_pre2021.sh --- tools/get_pre2021.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tools/get_pre2021.sh b/tools/get_pre2021.sh index b2e24565..764a92d9 100755 --- a/tools/get_pre2021.sh +++ b/tools/get_pre2021.sh @@ -10,11 +10,6 @@ RELEASE_DIR="gochan-${TESTING_VERSION}_linux" RELEASE_GZ="$RELEASE_DIR.tar.gz" RELEASE_URL="https://github.com/gochan-org/gochan/releases/download/$TESTING_VERSION/$RELEASE_GZ" -if [ -z "$STY" ]; then - echo "This command should be run from a screen instance" - echo "Example: screen -S get_pre2021 $0" - exit 1 -fi if [ "$USER" != "vagrant" ]; then echo "This must be run in the vagrant VM (expected \$USER to be vagrant, got $USER)" @@ -29,7 +24,7 @@ echo "Extracting $RELEASE_GZ" tar -xf gochan-v2.12.0_linux.tar.gz cd $RELEASE_DIR -cp examples/configs/gochan.example.json gochan.json +cp sample-configs/gochan.example.json gochan.json echo "Modifying $PWD/gochan.json for testing migration" sed -i gochan.json \ -e 's/"Port": .*/"Port": 9000,/' \ @@ -64,4 +59,4 @@ else exit 1 fi -sudo ./gochan \ No newline at end of file +sudo ./gochan From 75d5b32cdc4ae46710aa8a896da7e2e9ddc001e8 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 29 Dec 2024 17:49:05 -0800 Subject: [PATCH 024/122] Start working on making it possible to migrate old imageboard DB in place --- .../internal/common/handler.go | 5 +++++ .../internal/gcupdate/gcupdate.go | 14 ++++++++----- .../internal/pre2021/boards.go | 11 ++++++++++ .../internal/pre2021/pre2021.go | 15 +++++++------- cmd/gochan-migration/main.go | 20 +++++++++++++++---- pkg/gcsql/database.go | 3 +-- 6 files changed, 50 insertions(+), 18 deletions(-) diff --git a/cmd/gochan-migration/internal/common/handler.go b/cmd/gochan-migration/internal/common/handler.go index 3edb3815..fa92a541 100644 --- a/cmd/gochan-migration/internal/common/handler.go +++ b/cmd/gochan-migration/internal/common/handler.go @@ -56,6 +56,11 @@ type DBMigrator interface { // will exit IsMigrated() (bool, error) + // IsMigratingInPlace returns true if the old database name is the same as the new database name, + // meaning that the tables will be altered to match the new schema, instead of creating tables in the + // new database and copying data over + IsMigratingInPlace() bool + // MigrateDB alters the database schema to match the new schema, then migrates the imageboard // data (posts, boards, etc) to the new database. It is assumed that MigrateDB will handle // logging any errors that occur during the migration diff --git a/cmd/gochan-migration/internal/gcupdate/gcupdate.go b/cmd/gochan-migration/internal/gcupdate/gcupdate.go index ff2e0718..7e194781 100644 --- a/cmd/gochan-migration/internal/gcupdate/gcupdate.go +++ b/cmd/gochan-migration/internal/gcupdate/gcupdate.go @@ -2,7 +2,6 @@ package gcupdate import ( "context" - "errors" "fmt" "strings" "time" @@ -22,6 +21,11 @@ type GCDatabaseUpdater struct { TargetDBVer int } +// IsMigratingInPlace implements common.DBMigrator. +func (dbu *GCDatabaseUpdater) IsMigratingInPlace() bool { + return true +} + func (dbu *GCDatabaseUpdater) Init(options *common.MigrationOptions) error { dbu.options = options sqlCfg := config.GetSQLConfig() @@ -174,7 +178,7 @@ func (dbu *GCDatabaseUpdater) migrateFileBans(ctx context.Context, sqlConfig *co tx, err := dbu.db.BeginTx(ctx, nil) defer func() { if a := recover(); a != nil { - err = errors.New(fmt.Sprintf("recovered: %v", a)) + err = fmt.Errorf("recovered: %v", a) errEv.Caller(4).Err(err).Send() errEv.Discard() } else if err != nil { @@ -262,7 +266,7 @@ func (dbu *GCDatabaseUpdater) migrateFilenameBans(ctx context.Context, _ *config tx, err := dbu.db.BeginTx(ctx, nil) defer func() { if a := recover(); a != nil { - err = errors.New(fmt.Sprintf("recovered: %v", a)) + err = fmt.Errorf("recovered: %v", a) errEv.Caller(4).Err(err).Send() errEv.Discard() } else if err != nil { @@ -320,7 +324,7 @@ func (dbu *GCDatabaseUpdater) migrateUsernameBans(ctx context.Context, _ *config tx, err := dbu.db.BeginTx(ctx, nil) defer func() { if a := recover(); a != nil { - err = errors.New(fmt.Sprintf("recovered: %v", a)) + err = fmt.Errorf("recovered: %v", a) errEv.Caller(4).Err(err).Send() errEv.Discard() } else if err != nil { @@ -378,7 +382,7 @@ func (dbu *GCDatabaseUpdater) migrateWordfilters(ctx context.Context, sqlConfig tx, err := dbu.db.BeginTx(ctx, nil) defer func() { if a := recover(); a != nil { - err = errors.New(fmt.Sprintf("recovered: %v", a)) + err = fmt.Errorf("recovered: %v", a) errEv.Caller(4).Err(err).Send() errEv.Discard() } else if err != nil { diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index c5aa1c20..6497c34c 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -1,12 +1,19 @@ package pre2021 import ( + "fmt" "log" + "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/gcsql" ) func (m *Pre2021Migrator) MigrateBoards() error { + defer func() { + if r := recover(); r != nil { + common.LogFatal().Interface("recover", r).Msg("Recovered from panic in MigrateBoards") + } + }() if m.oldBoards == nil { m.oldBoards = map[int]string{} } @@ -14,10 +21,14 @@ func (m *Pre2021Migrator) MigrateBoards() error { m.newBoards = map[int]string{} } // get all boards from new db + fmt.Println("0") err := gcsql.ResetBoardSectionArrays() + fmt.Println("1") if err != nil { + fmt.Println("2") return nil } + fmt.Println("3") // get boards from old db rows, err := m.db.QuerySQL(boardsQuery) diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index 0d3ffcc1..ae5ebdd0 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -12,12 +12,6 @@ import ( type Pre2021Config struct { config.SQLConfig - // DBtype string - // DBhost string - // DBname string - // DBusername string - // DBpassword string - // DBprefix string DocumentRoot string } @@ -31,6 +25,11 @@ type Pre2021Migrator struct { newBoards map[int]string // map[board]dir } +// IsMigratingInPlace implements common.DBMigrator. +func (m *Pre2021Migrator) IsMigratingInPlace() bool { + return m.config.DBname == config.GetSQLConfig().DBname +} + func (m *Pre2021Migrator) readConfig() error { ba, err := os.ReadFile(m.options.OldChanConfig) if err != nil { @@ -68,7 +67,7 @@ func (m *Pre2021Migrator) IsMigrated() (bool, error) { return false, gcsql.ErrUnsupportedDB } if err = m.db.QueryRowSQL(query, - []interface{}{m.config.DBprefix + "migrated", m.config.DBname}, + []interface{}{m.config.DBprefix + "database_version", m.config.DBname}, []interface{}{&migrated}); err != nil { return migrated, err } @@ -82,12 +81,14 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) { } if migrated { // db is already migrated, stop + common.LogWarning().Msg("Database is already migrated (database_version table exists)") return true, nil } if err := m.MigrateBoards(); err != nil { return false, err } + common.LogInfo().Msg("Migrated boards") // if err = m.MigratePosts(); err != nil { // return false, err // } diff --git a/cmd/gochan-migration/main.go b/cmd/gochan-migration/main.go index bf84f45a..c52e87fd 100644 --- a/cmd/gochan-migration/main.go +++ b/cmd/gochan-migration/main.go @@ -56,9 +56,14 @@ func main() { fatalEv.Discard() }() - if !updateDB && (options.ChanType == "" || options.OldChanConfig == "") { - flag.PrintDefaults() - fatalEv.Msg("Missing required oldchan value") + if !updateDB { + if options.ChanType == "" { + flag.PrintDefaults() + fatalEv.Msg("Missing required oldchan value") + } else if options.OldChanConfig == "" { + flag.PrintDefaults() + fatalEv.Msg("Missing required oldconfig value") + } } else if updateDB { options.ChanType = "gcupdate" } @@ -82,8 +87,15 @@ func main() { default: fatalEv.Msg("Unsupported chan type, Currently only pre2021 database migration is supported") } + migratingInPlace := migrator.IsMigratingInPlace() + common.LogInfo(). + Str("oldChanType", options.ChanType). + Str("oldChanConfig", options.OldChanConfig). + Bool("migratingInPlace", migratingInPlace). + Msg("Starting database migration") + config.InitConfig(versionStr) - if !updateDB { + if !migratingInPlace { sqlCfg := config.GetSQLConfig() err = gcsql.ConnectToDB(&sqlCfg) if err != nil { diff --git a/pkg/gcsql/database.go b/pkg/gcsql/database.go index 163c5554..f9cdb51a 100644 --- a/pkg/gcsql/database.go +++ b/pkg/gcsql/database.go @@ -416,8 +416,7 @@ func SetupMockDB(driver string) (sqlmock.Sqlmock, error) { return mock, err } -// Open opens and returns a new gochan database connection with the provided host, driver, DB name, -// username, password, and table prefix +// Open opens and returns a new gochan database connection with the provided SQL options func Open(cfg *config.SQLConfig) (db *GCDB, err error) { db, err = setupDBConn(cfg) if err != nil { From 06321bb5fd2f145a61f6eb40ad19ed05df207099 Mon Sep 17 00:00:00 2001 From: "o.nihilist" <107046146+onihilist@users.noreply.github.com> Date: Tue, 31 Dec 2024 09:40:01 +0100 Subject: [PATCH 025/122] Footer & Header Templates Tests --- pkg/gctemplates/templatetests/templates_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/gctemplates/templatetests/templates_test.go b/pkg/gctemplates/templatetests/templates_test.go index d8ce11c7..8c088bba 100644 --- a/pkg/gctemplates/templatetests/templates_test.go +++ b/pkg/gctemplates/templatetests/templates_test.go @@ -108,3 +108,11 @@ func TestTemplateBase(t *testing.T) { initTemplatesMock(t, mock) } + +func TestBaseFooter(t *testing.T) { + runTemplateTestCases(t, gctemplates.PageFooter, baseFooterCases) +} + +func TestBaseHeader(t *testing.T) { + runTemplateTestCases(t, gctemplates.PageHeader, baseHeaderCases) +} From 6b5498cb7aa063d7f5d9e56dd1be5d69841c7bc3 Mon Sep 17 00:00:00 2001 From: "o.nihilist" <107046146+onihilist@users.noreply.github.com> Date: Tue, 31 Dec 2024 09:40:29 +0100 Subject: [PATCH 026/122] Footer & Header Templates Tests --- .../templatetests/templatecases_test.go | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) diff --git a/pkg/gctemplates/templatetests/templatecases_test.go b/pkg/gctemplates/templatetests/templatecases_test.go index affaf46a..740ed489 100644 --- a/pkg/gctemplates/templatetests/templatecases_test.go +++ b/pkg/gctemplates/templatetests/templatecases_test.go @@ -70,6 +70,18 @@ var ( AnonymousName: "Anonymous Coward", } + simpleBoard2 = &gcsql.Board{ + ID: 2, + SectionID: 2, + URI: "sup", + Dir: "sup", + Title: "Gochan Support board", + Subtitle: "Board for helping out gochan users/admins", + Description: "Board for helping out gochan users/admins", + DefaultStyle: "yotsuba.css", + AnonymousName: "Anonymous Coward", + } + banPageCases = []templateTestCase{ { desc: "appealable permaban", @@ -251,6 +263,195 @@ var ( expectedOutput: `const styles=[];const defaultStyle="\"a\\a\"";const webroot="";const serverTZ=0;const fileTypes=[];`, }, } + + baseFooterCases = []templateTestCase{ + { + desc: "base footer test", + data: map[string]any{ + "boardConfig": simpleBoardConfig, + "board": simpleBoard1, + "numPages": 1, + "sections": []gcsql.Section{ + {ID: 1}, + }, + }, + expectedOutput: footer, + }, + { + desc: "base footer test", + data: map[string]any{ + "boardConfig": simpleBoardConfig, + "board": simpleBoard2, + "numPages": 3, + "sections": []gcsql.Section{ + {ID: 1}, + }, + }, + expectedOutput: footer, + }, + } + + baseHeaderCases = []templateTestCase{ + { + desc: "Header Test /test/", + data: map[string]any{ + "boardConfig": simpleBoardConfig, + "board": simpleBoard1, + "numPages": 1, + "sections": []gcsql.Section{ + {ID: 1}, + }, + }, + expectedOutput: headBeginning + + `/test/-Testing board` + + `` + + `` + + `` + + `` + + `` + + `
` + + `
`, + }, + { + desc: "Header Test /sup/", + data: map[string]any{ + "boardConfig": simpleBoardConfig, + "board": simpleBoard2, + "numPages": 1, + "sections": []gcsql.Section{ + {ID: 1}, + }, + }, + expectedOutput: headBeginning + + `/sup/-Gochan Support board` + + `` + + `` + + `` + + `` + + `` + + `
` + + `home
` + + `
` + + `
`, + }, + { + desc: "Perma Ban Header Test", + data: map[string]any{ + "ban": &gcsql.IPBan{ + RangeStart: "192.168.56.0", + RangeEnd: "192.168.56.255", + IPBanBase: gcsql.IPBanBase{ + IsActive: true, + Permanent: true, + StaffID: 1, + Message: "ban message goes here", + }, + }, + "ip": "192.168.56.1", + "siteConfig": testingSiteConfig, + "systemCritical": config.SystemCriticalConfig{ + WebRoot: "/", + }, + "boardConfig": config.BoardConfig{ + DefaultStyle: "pipes.css", + }, + }, + expectedOutput: `` + + `` + + `YOU'RE PERMABANNED, IDIOT!` + + `` + + `` + + `
`, + }, + { + desc: "Appealable Perma Ban Header Test", + data: map[string]any{ + "ban": &gcsql.IPBan{ + RangeStart: "192.168.56.0", + RangeEnd: "192.168.56.255", + IPBanBase: gcsql.IPBanBase{ + Permanent: true, + CanAppeal: true, + StaffID: 1, + Message: "ban message goes here", + }, + }, + "ip": "192.168.56.1", + "siteConfig": testingSiteConfig, + "systemCritical": config.SystemCriticalConfig{ + WebRoot: "/", + }, + "boardConfig": config.BoardConfig{ + DefaultStyle: "pipes.css", + }, + }, + expectedOutput: `` + + `` + + `YOU ARE BANNED:(` + + `` + + `` + + `
`, + }, + { + desc: "Appealable Temp Ban Header Test", + data: map[string]any{ + "ban": &gcsql.IPBan{ + RangeStart: "192.168.56.0", + RangeEnd: "192.168.56.255", + IPBanBase: gcsql.IPBanBase{ + CanAppeal: true, + StaffID: 1, + Message: "ban message goes here", + }, + }, + "ip": "192.168.56.1", + "siteConfig": testingSiteConfig, + "systemCritical": config.SystemCriticalConfig{ + WebRoot: "/", + }, + "boardConfig": config.BoardConfig{ + DefaultStyle: "pipes.css", + }, + }, + expectedOutput: `` + + `` + + `YOU ARE BANNED:(` + + `` + + `` + + `
`, + }, + { + desc: "Unappealable Temp Ban Header Test", + data: map[string]any{ + "ban": &gcsql.IPBan{ + RangeStart: "192.168.56.0", + RangeEnd: "192.168.56.255", + IPBanBase: gcsql.IPBanBase{ + StaffID: 1, + Message: "ban message goes here", + }, + }, + "ip": "192.168.56.1", + "siteConfig": testingSiteConfig, + "systemCritical": config.SystemCriticalConfig{ + WebRoot: "/", + }, + "boardConfig": config.BoardConfig{ + DefaultStyle: "pipes.css", + }, + }, + expectedOutput: `` + + `` + + `YOU ARE BANNED:(` + + `` + + `` + + `
`, + }, + } ) const ( From 0d0aca83afb1f92ac66c64c1a437be18bbc8670b Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Tue, 31 Dec 2024 15:36:20 -0800 Subject: [PATCH 027/122] Add gcsql.GetSectionFromName and start overhaul of pre2021 migration --- .../internal/common/handler.go | 1 - .../internal/pre2021/boards.go | 258 ++++++++++-------- .../internal/pre2021/posts.go | 24 +- .../internal/pre2021/pre2021.go | 7 +- .../internal/pre2021/queries.go | 62 +---- pkg/gcsql/sections.go | 24 +- 6 files changed, 202 insertions(+), 174 deletions(-) diff --git a/cmd/gochan-migration/internal/common/handler.go b/cmd/gochan-migration/internal/common/handler.go index fa92a541..fa3e64ee 100644 --- a/cmd/gochan-migration/internal/common/handler.go +++ b/cmd/gochan-migration/internal/common/handler.go @@ -41,7 +41,6 @@ type MigrationOptions struct { OldChanConfig string OldDBName string NewDBName string - DirAction int } // DBMigrator is used for handling the migration from one database type to a diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index 6497c34c..4c44ac84 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -1,92 +1,139 @@ package pre2021 import ( - "fmt" - "log" + "errors" + "time" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/gcsql" ) -func (m *Pre2021Migrator) MigrateBoards() error { - defer func() { - if r := recover(); r != nil { - common.LogFatal().Interface("recover", r).Msg("Recovered from panic in MigrateBoards") - } - }() - if m.oldBoards == nil { - m.oldBoards = map[int]string{} - } - if m.newBoards == nil { - m.newBoards = map[int]string{} - } - // get all boards from new db - fmt.Println("0") - err := gcsql.ResetBoardSectionArrays() - fmt.Println("1") - if err != nil { - fmt.Println("2") - return nil - } - fmt.Println("3") +type boardTable struct { + id int + listOrder int + dir string + boardType int + uploadType int + title string + subtitle string + description string + section int + maxFileSize int + maxPages int + defaultStyle string + locked bool + createdOn time.Time + anonymous string + forcedAnon bool + maxAge int + autosageAfter int + noImagesAfter int + maxMessageLength int + embedsAllowed bool + redirectToThread bool + requireFile bool + enableCatalog bool +} - // get boards from old db - rows, err := m.db.QuerySQL(boardsQuery) +func (m *Pre2021Migrator) migrateBoardsInPlace() error { + return nil +} + +func (m *Pre2021Migrator) createSectionIfNotExist(sectionCheck *gcsql.Section) (int, error) { + // to be used when not migrating in place, otherwise the section table should be altered + section, err := gcsql.GetSectionFromName(sectionCheck.Name) + if errors.Is(err, gcsql.ErrSectionDoesNotExist) { + // section doesn't exist, create it + section, err = gcsql.NewSection(sectionCheck.Name, section.Abbreviation, section.Hidden, section.Position) + if err != nil { + return 0, err + } + } + return section.ID, nil +} + +func (m *Pre2021Migrator) migrateSectionsToNewDB() error { + // creates sections in the new db if they don't exist, and also creates a migration section that + // boards will be set to, to be moved to the correct section by the admin after migration + rows, err := m.db.QuerySQL(sectionsQuery) if err != nil { return err } defer rows.Close() + for rows.Next() { - var id int - var dir string - var title string - var subtitle string - var description string - var section int - var max_file_size int - var max_pages int - var default_style string - var locked bool - var anonymous string - var forced_anon bool - var max_age int - var autosage_after int - var no_images_after int - var max_message_length int - var embeds_allowed bool - var redirect_to_thread bool - var require_file bool - var enable_catalog bool + var section gcsql.Section if err = rows.Scan( - &id, - &dir, - &title, - &subtitle, - &description, - §ion, - &max_file_size, - &max_pages, - &default_style, - &locked, - &anonymous, - &forced_anon, - &max_age, - &autosage_after, - &no_images_after, - &max_message_length, - &embeds_allowed, - &redirect_to_thread, - &require_file, - &enable_catalog); err != nil { + §ion.ID, + §ion.Position, + §ion.Hidden, + §ion.Name, + §ion.Abbreviation, + ); err != nil { + return err + } + if _, err = m.createSectionIfNotExist(§ion); err != nil { + return err + } + } + if err = rows.Close(); err != nil { + return err + } + m.migrationSectionID, err = m.createSectionIfNotExist(&gcsql.Section{ + Name: "Migrated Boards", + Abbreviation: "mb", + Hidden: true, + }) + + return err +} + +func (m *Pre2021Migrator) migrateBoardsToNewDB() error { + if m.oldBoards == nil { + m.oldBoards = make(map[string]boardTable) + } + if m.newBoards == nil { + m.newBoards = make(map[string]boardTable) + } + errEv := common.LogError() + defer errEv.Discard() + + err := m.migrateSectionsToNewDB() + if err != nil { + errEv.Err(err).Msg("Failed to migrate sections") + } + + // get all boards from new db + if err = gcsql.ResetBoardSectionArrays(); err != nil { + return nil + } + + // get boards from old db + rows, err := m.db.QuerySQL(boardsQuery) + if err != nil { + errEv.Err(err).Caller().Msg("Failed to query old database boards") + return err + } + defer rows.Close() + for rows.Next() { + var board boardTable + if err = rows.Scan( + &board.id, &board.listOrder, &board.dir, &board.boardType, &board.uploadType, &board.title, &board.subtitle, + &board.description, &board.section, &board.maxFileSize, &board.maxPages, &board.defaultStyle, &board.locked, + &board.createdOn, &board.anonymous, &board.forcedAnon, &board.maxAge, &board.autosageAfter, &board.noImagesAfter, + &board.maxMessageLength, &board.embedsAllowed, &board.redirectToThread, &board.requireFile, &board.enableCatalog, + ); err != nil { + errEv.Err(err).Caller().Msg("Failed to scan row into board") return err } found := false - for b := range gcsql.AllBoards { - if _, ok := m.oldBoards[id]; !ok { - m.oldBoards[id] = dir + for _, newBoard := range gcsql.AllBoards { + if _, ok := m.oldBoards[board.dir]; !ok { + m.oldBoards[board.dir] = board } - if gcsql.AllBoards[b].Dir == dir { - log.Printf("Board /%s/ already exists in new db, moving on\n", dir) + + if newBoard.Dir == board.dir { + common.LogWarning().Str("board", board.dir).Msg("Board already exists in new db, moving on") found = true break } @@ -97,48 +144,41 @@ func (m *Pre2021Migrator) MigrateBoards() error { // create new board using the board data from the old db // omitting things like ID and creation date since we don't really care if err = gcsql.CreateBoard(&gcsql.Board{ - Dir: dir, - Title: title, - Subtitle: subtitle, - Description: description, - SectionID: section, - MaxFilesize: max_file_size, - // MaxPages: max_pages, - DefaultStyle: default_style, - Locked: locked, - AnonymousName: anonymous, - ForceAnonymous: forced_anon, - // MaxAge: max_age, - AutosageAfter: autosage_after, - NoImagesAfter: no_images_after, - MaxMessageLength: max_message_length, - AllowEmbeds: embeds_allowed, - RedirectToThread: redirect_to_thread, - RequireFile: require_file, - EnableCatalog: enable_catalog, + Dir: board.dir, + Title: board.title, + Subtitle: board.subtitle, + Description: board.description, + SectionID: board.section, + MaxFilesize: board.maxFileSize, + DefaultStyle: board.defaultStyle, + Locked: board.locked, + AnonymousName: board.anonymous, + ForceAnonymous: board.forcedAnon, + AutosageAfter: board.autosageAfter, + NoImagesAfter: board.noImagesAfter, + MaxMessageLength: board.maxMessageLength, + AllowEmbeds: board.embedsAllowed, + RedirectToThread: board.redirectToThread, + RequireFile: board.requireFile, + EnableCatalog: board.enableCatalog, }, false); err != nil { + errEv.Err(err).Caller().Str("board", board.dir).Msg("Failed to create board") return err } - m.newBoards[id] = dir - log.Printf("/%s/ successfully migrated in the database", dir) - // Automatic directory migration has the potential to go horribly wrong, so I'm leaving this - // commented out for now - // switch m.options.DirAction { - // case common.DirCopy: - - // case common.DirMove: - // // move the old directory (probably should copy instead) to the new one - // newDocumentRoot := config.GetSystemCriticalConfig().DocumentRoot - // log.Println("Old board path:", path.Join(m.config.DocumentRoot, dir)) - // log.Println("Old board path:", path.Join(newDocumentRoot, dir)) - // if err = os.Rename( - // path.Join(m.config.DocumentRoot, dir), - // path.Join(newDocumentRoot, dir), - // ); err != nil { - // return err - // } - // log.Printf("/%s/ directory/files successfully moved") - // } + m.newBoards[board.dir] = board + common.LogInfo().Str("board", board.dir).Msg("Board successfully migrated") } return nil } + +func (m *Pre2021Migrator) MigrateBoards() error { + defer func() { + if r := recover(); r != nil { + common.LogFatal().Interface("recover", r).Msg("Recovered from panic in MigrateBoards") + } + }() + if m.IsMigratingInPlace() { + return m.migrateBoardsInPlace() + } + return m.migrateBoardsToNewDB() +} diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index 8398627b..2318d111 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -3,9 +3,9 @@ package pre2021 import ( "database/sql" "fmt" - "log" "time" + "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/gcsql" ) @@ -39,6 +39,7 @@ type postTable struct { reviewed bool newBoardID int + foundBoard bool // oldParentID int } @@ -102,12 +103,21 @@ func (m *Pre2021Migrator) migrateThreads() error { tx.Rollback() return err } - _, ok := m.oldBoards[post.boardid] - if !ok { - // board doesn't exist - log.Printf( - "Pre-migrated post #%d has an invalid boardid %d (board doesn't exist), skipping", - post.id, post.boardid) + var postBoardDir string + for _, oldBoard := range m.oldBoards { + if oldBoard.id == post.boardid { + postBoardDir = oldBoard.dir + } + } + for _, newBoard := range gcsql.AllBoards { + if newBoard.Dir == postBoardDir { + post.newBoardID = newBoard.ID + post.foundBoard = true + } + } + if !post.foundBoard { + common.LogWarning().Int("boardID", post.boardid). + Msg("Pre-migrated post has an invalid boardid (board doesn't exist), skipping") continue } diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index ae5ebdd0..3bd53a6c 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -20,9 +20,10 @@ type Pre2021Migrator struct { options *common.MigrationOptions config Pre2021Config - posts []postTable - oldBoards map[int]string // map[boardid]dir - newBoards map[int]string // map[board]dir + migrationSectionID int + posts []postTable + oldBoards map[string]boardTable + newBoards map[string]boardTable } // IsMigratingInPlace implements common.DBMigrator. diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index 6482a7be..b0c49809 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -1,57 +1,15 @@ package pre2021 const ( - boardsQuery = `SELECT - id, - dir, - type, - upload_type, - title, - subtitle, - description, - section, - max_file_size, - max_pages, - default_style, - locked, - anonymous, - forced_anon, - max_age, - autosage_after, - no_images_after, - max_message_length, - embeds_allowed, - redirect_to_thread, - require_file, - enable_catalog - FROM DBPREFIXboards` + sectionsQuery = `SELECT id, list_order, hidden, name, abbreviation FROM DBPREFIXsections` - postsQuery = `SELECT - id, - boardid, - parentid, - name, - tripcode, - email, - subject, - message, - message_raw, - password, - filename, - filename_original, - file_checksum, - filesize, - image_w, - image_h, - thumb_w, - thumb_h, - ip, - tag, - timestamp, - autosage, - deleted_timestamp, - bumped, - stickied, - locked, - reviewed from DBPREFIXposts WHERE deleted_timestamp = NULL` + boardsQuery = `SELECT id, list_order, dir, type, upload_type, title, subtitle, description, section, max_file_size, max_pages, +default_style, locked, created_on, anonymous, forced_anon, max_age, autosage_after, no_images_after, max_message_length, embeds_allowed, +redirect_to_thread, require_file, enable_catalog +FROM DBPREFIXboards` + + postsQuery = `SELECT id, boardid, parentid, name, tripcode, email, subject, message, message_raw, password, filename, +filename_original, file_checksum, filesize, image_w, image_h, thumb_w, thumb_h, ip, tag, timestamp, autosage, deleted_timestamp, +bumped, stickied, locked, reviewed +FROM DBPREFIXposts WHERE deleted_timestamp = NULL` ) diff --git a/pkg/gcsql/sections.go b/pkg/gcsql/sections.go index 50db9f72..18ab4037 100644 --- a/pkg/gcsql/sections.go +++ b/pkg/gcsql/sections.go @@ -9,7 +9,8 @@ import ( var ( // AllSections provides a quick and simple way to access a list of all non-hidden sections without // having to do any SQL queries. It and AllBoards are updated by ResetBoardSectionArrays - AllSections []Section + AllSections []Section + ErrSectionDoesNotExist = errors.New("section does not exist") ) // GetAllSections gets a list of all existing sections, optionally omitting hidden ones @@ -62,18 +63,37 @@ func getOrCreateDefaultSectionID() (sectionID int, err error) { return id, nil } +// GetSectionFromID returns a section from the database, given its ID func GetSectionFromID(id int) (*Section, error) { const query = `SELECT id, name, abbreviation, position, hidden FROM DBPREFIXsections WHERE id = ?` var section Section err := QueryRowTimeoutSQL(nil, query, []any{id}, []any{ §ion.ID, §ion.Name, §ion.Abbreviation, §ion.Position, §ion.Hidden, }) - if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrSectionDoesNotExist + } else if err != nil { return nil, err } return §ion, err } +// GetSectionFromID returns a section from the database, given its name +func GetSectionFromName(name string) (*Section, error) { + const query = `SELECT id, name, abbreviation, position, hidden FROM DBPREFIXsections WHERE name = ?` + var section Section + err := QueryRowTimeoutSQL(nil, query, []any{name}, []any{ + §ion.ID, §ion.Name, §ion.Abbreviation, §ion.Position, §ion.Hidden, + }) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrSectionDoesNotExist + } else if err != nil { + return nil, err + } + return §ion, err +} + +// DeleteSection deletes a section from the database and resets the AllSections array func DeleteSection(id int) error { const query = `DELETE FROM DBPREFIXsections WHERE id = ?` _, err := ExecSQL(query, id) From bdf4bddc4d8985375988f8adaf34261ccb28c88c Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Tue, 31 Dec 2024 15:46:02 -0800 Subject: [PATCH 028/122] Bump version to 4.0.2 for hotfix release Resolves issue #112 --- build.py | 2 +- frontend/package.json | 2 +- html/error/404.html | 2 +- html/error/500.html | 2 +- html/error/502.html | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.py b/build.py index 22eebbf8..c5fbe2dd 100755 --- a/build.py +++ b/build.py @@ -39,7 +39,7 @@ release_files = ( "README.md", ) -GOCHAN_VERSION = "4.0.1" +GOCHAN_VERSION = "4.0.2" DATABASE_VERSION = "4" # stored in DBNAME.DBPREFIXdatabase_version PATH_NOTHING = -1 diff --git a/frontend/package.json b/frontend/package.json index fff7198c..22fc29b8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "gochan.js", - "version": "4.0.1", + "version": "4.0.2", "description": "", "main": "./ts/main.ts", "private": true, diff --git a/html/error/404.html b/html/error/404.html index 7299fdc1..602780fd 100755 --- a/html/error/404.html +++ b/html/error/404.html @@ -7,6 +7,6 @@

404: File not found

lol 404

The requested file could not be found on this server.

-
Site powered by Gochan v4.0.1 +
Site powered by Gochan v4.0.2 \ No newline at end of file diff --git a/html/error/500.html b/html/error/500.html index 6578dbd6..8fe12c28 100755 --- a/html/error/500.html +++ b/html/error/500.html @@ -7,6 +7,6 @@

Error 500: Internal Server error

server burning

The server encountered an error while trying to serve the page, and we apologize for the inconvenience. The system administrator will try to fix things as soon they get around to it, whenever that is. Hopefully soon.

-
Site powered by Gochan v4.0.1 +
Site powered by Gochan v4.0.2 \ No newline at end of file diff --git a/html/error/502.html b/html/error/502.html index 651b5634..e5644dd5 100644 --- a/html/error/502.html +++ b/html/error/502.html @@ -7,6 +7,6 @@

Error 502: Bad gateway

server burning

The server encountered an error while trying to serve the page, and we apologize for the inconvenience. The system administrator will try to fix things as soon they get around to it, whenever that is. Hopefully soon.

-
Site powered by Gochan v4.0.1 +
Site powered by Gochan v4.0.2 \ No newline at end of file From 3d15c272e910ce0362e82df2de471cbcb6e0b45a Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Tue, 31 Dec 2024 17:16:25 -0800 Subject: [PATCH 029/122] Include stack trace in MigrateBoards recovery log, replace old stuff in IsMigrated with common.TableExists --- .../internal/pre2021/boards.go | 16 +++++++-- .../internal/pre2021/pre2021.go | 34 +++++++------------ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index 4c44ac84..a1c7770f 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -2,10 +2,13 @@ package pre2021 import ( "errors" + "runtime/debug" + "strings" "time" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/gcsql" + "github.com/rs/zerolog" ) type boardTable struct { @@ -44,7 +47,7 @@ func (m *Pre2021Migrator) createSectionIfNotExist(sectionCheck *gcsql.Section) ( section, err := gcsql.GetSectionFromName(sectionCheck.Name) if errors.Is(err, gcsql.ErrSectionDoesNotExist) { // section doesn't exist, create it - section, err = gcsql.NewSection(sectionCheck.Name, section.Abbreviation, section.Hidden, section.Position) + section, err = gcsql.NewSection(sectionCheck.Name, section.Abbreviation, true, 0) if err != nil { return 0, err } @@ -174,7 +177,16 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { func (m *Pre2021Migrator) MigrateBoards() error { defer func() { if r := recover(); r != nil { - common.LogFatal().Interface("recover", r).Msg("Recovered from panic in MigrateBoards") + stackTrace := debug.Stack() + traceLines := strings.Split(string(stackTrace), "\n") + zlArr := zerolog.Arr() + for _, line := range traceLines { + zlArr.Str(line) + } + common.LogFatal().Caller(). + Interface("recover", r). + Array("stackTrace", zlArr). + Msg("Recovered from panic in MigrateBoards") } }() if m.IsMigratingInPlace() { diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index 3bd53a6c..6d5615eb 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -2,8 +2,10 @@ package pre2021 import ( + "context" "encoding/json" "os" + "time" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/config" @@ -24,6 +26,7 @@ type Pre2021Migrator struct { posts []postTable oldBoards map[string]boardTable newBoards map[string]boardTable + threads map[int]int // old thread id (previously stored in posts as the id) to new thread id (threads.id) } // IsMigratingInPlace implements common.DBMigrator. @@ -36,48 +39,34 @@ func (m *Pre2021Migrator) readConfig() error { if err != nil { return err } + m.config.SQLConfig = config.GetSQLConfig() return json.Unmarshal(ba, &m.config) } func (m *Pre2021Migrator) Init(options *common.MigrationOptions) error { m.options = options var err error + m.config.SQLConfig = config.GetSQLConfig() if err = m.readConfig(); err != nil { return err } - m.config.DBTimeoutSeconds = config.DefaultSQLTimeout - m.config.DBMaxOpenConnections = config.DefaultSQLMaxConns - m.config.DBMaxIdleConnections = config.DefaultSQLMaxConns - m.config.DBConnMaxLifetimeMin = config.DefaultSQLConnMaxLifetimeMin m.db, err = gcsql.Open(&m.config.SQLConfig) return err } func (m *Pre2021Migrator) IsMigrated() (bool, error) { - var migrated bool - var err error - var query string - switch m.config.DBtype { - case "mysql": - fallthrough - case "postgres": - query = `SELECT COUNT(*) > 0 FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_NAME = ? AND TABLE_SCHEMA = ?` - default: - return false, gcsql.ErrUnsupportedDB - } - if err = m.db.QueryRowSQL(query, - []interface{}{m.config.DBprefix + "database_version", m.config.DBname}, - []interface{}{&migrated}); err != nil { - return migrated, err - } - return migrated, err + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(m.config.DBTimeoutSeconds)*time.Second) + defer cancel() + return common.TableExists(ctx, m.db, nil, "DBPREFIXdatabase_version", &m.config.SQLConfig) } func (m *Pre2021Migrator) MigrateDB() (bool, error) { + errEv := common.LogError() + defer errEv.Discard() migrated, err := m.IsMigrated() if err != nil { + errEv.Caller().Err(err).Msg("Error checking if database is migrated") return false, err } if migrated { @@ -87,6 +76,7 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) { } if err := m.MigrateBoards(); err != nil { + errEv.Caller().Err(err).Msg("Failed to migrate boards") return false, err } common.LogInfo().Msg("Migrated boards") From 8b994dce5736483073b616112ef5b029375b30b3 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 1 Jan 2025 11:55:44 -0800 Subject: [PATCH 030/122] Consolidate board fields in pre-2021 migration struct and use equivalent gcsql.Board fields --- .../internal/pre2021/boards.go | 86 +++++-------------- .../internal/pre2021/posts.go | 5 -- .../internal/pre2021/pre2021.go | 5 +- .../internal/pre2021/queries.go | 4 +- 4 files changed, 26 insertions(+), 74 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index a1c7770f..da483068 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -4,38 +4,16 @@ import ( "errors" "runtime/debug" "strings" - "time" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" + "github.com/gochan-org/gochan/pkg/config" "github.com/gochan-org/gochan/pkg/gcsql" "github.com/rs/zerolog" ) -type boardTable struct { - id int - listOrder int - dir string - boardType int - uploadType int - title string - subtitle string - description string - section int - maxFileSize int - maxPages int - defaultStyle string - locked bool - createdOn time.Time - anonymous string - forcedAnon bool - maxAge int - autosageAfter int - noImagesAfter int - maxMessageLength int - embedsAllowed bool - redirectToThread bool - requireFile bool - enableCatalog bool +type migrationBoard struct { + oldID int + gcsql.Board } func (m *Pre2021Migrator) migrateBoardsInPlace() error { @@ -47,7 +25,7 @@ func (m *Pre2021Migrator) createSectionIfNotExist(sectionCheck *gcsql.Section) ( section, err := gcsql.GetSectionFromName(sectionCheck.Name) if errors.Is(err, gcsql.ErrSectionDoesNotExist) { // section doesn't exist, create it - section, err = gcsql.NewSection(sectionCheck.Name, section.Abbreviation, true, 0) + section, err = gcsql.NewSection(sectionCheck.Name, sectionCheck.Abbreviation, true, 0) if err != nil { return 0, err } @@ -92,11 +70,8 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error { } func (m *Pre2021Migrator) migrateBoardsToNewDB() error { - if m.oldBoards == nil { - m.oldBoards = make(map[string]boardTable) - } - if m.newBoards == nil { - m.newBoards = make(map[string]boardTable) + if m.boards == nil { + m.boards = make(map[string]migrationBoard) } errEv := common.LogError() defer errEv.Discard() @@ -119,57 +94,40 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { } defer rows.Close() for rows.Next() { - var board boardTable + var board migrationBoard + var maxPages int if err = rows.Scan( - &board.id, &board.listOrder, &board.dir, &board.boardType, &board.uploadType, &board.title, &board.subtitle, - &board.description, &board.section, &board.maxFileSize, &board.maxPages, &board.defaultStyle, &board.locked, - &board.createdOn, &board.anonymous, &board.forcedAnon, &board.maxAge, &board.autosageAfter, &board.noImagesAfter, - &board.maxMessageLength, &board.embedsAllowed, &board.redirectToThread, &board.requireFile, &board.enableCatalog, + &board.ID, &board.NavbarPosition, &board.Dir, &board.Title, &board.Subtitle, + &board.Description, &board.SectionID, &board.MaxFilesize, &maxPages, &board.DefaultStyle, &board.Locked, + &board.CreatedAt, &board.AnonymousName, &board.ForceAnonymous, &board.AutosageAfter, &board.NoImagesAfter, + &board.MaxMessageLength, &board.AllowEmbeds, &board.RedirectToThread, &board.RequireFile, &board.EnableCatalog, ); err != nil { errEv.Err(err).Caller().Msg("Failed to scan row into board") return err } + board.MaxThreads = maxPages * config.GetBoardConfig(board.Dir).ThreadsPerPage found := false for _, newBoard := range gcsql.AllBoards { - if _, ok := m.oldBoards[board.dir]; !ok { - m.oldBoards[board.dir] = board + if _, ok := m.boards[board.Dir]; !ok { + m.boards[board.Dir] = board } - - if newBoard.Dir == board.dir { - common.LogWarning().Str("board", board.dir).Msg("Board already exists in new db, moving on") + if newBoard.Dir == board.Dir { + common.LogWarning().Str("board", board.Dir).Msg("Board already exists in new db, moving on") found = true break } } + m.boards[board.Dir] = board if found { continue } // create new board using the board data from the old db // omitting things like ID and creation date since we don't really care - if err = gcsql.CreateBoard(&gcsql.Board{ - Dir: board.dir, - Title: board.title, - Subtitle: board.subtitle, - Description: board.description, - SectionID: board.section, - MaxFilesize: board.maxFileSize, - DefaultStyle: board.defaultStyle, - Locked: board.locked, - AnonymousName: board.anonymous, - ForceAnonymous: board.forcedAnon, - AutosageAfter: board.autosageAfter, - NoImagesAfter: board.noImagesAfter, - MaxMessageLength: board.maxMessageLength, - AllowEmbeds: board.embedsAllowed, - RedirectToThread: board.redirectToThread, - RequireFile: board.requireFile, - EnableCatalog: board.enableCatalog, - }, false); err != nil { - errEv.Err(err).Caller().Str("board", board.dir).Msg("Failed to create board") + if err = gcsql.CreateBoard(&board.Board, false); err != nil { + errEv.Err(err).Caller().Str("board", board.Dir).Msg("Failed to create board") return err } - m.newBoards[board.dir] = board - common.LogInfo().Str("board", board.dir).Msg("Board successfully migrated") + common.LogInfo().Str("board", board.Dir).Msg("Board successfully created") } return nil } diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index 2318d111..751dbf4f 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -104,11 +104,6 @@ func (m *Pre2021Migrator) migrateThreads() error { return err } var postBoardDir string - for _, oldBoard := range m.oldBoards { - if oldBoard.id == post.boardid { - postBoardDir = oldBoard.dir - } - } for _, newBoard := range gcsql.AllBoards { if newBoard.Dir == postBoardDir { post.newBoardID = newBoard.ID diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index 6d5615eb..3d04f99c 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -24,9 +24,8 @@ type Pre2021Migrator struct { migrationSectionID int posts []postTable - oldBoards map[string]boardTable - newBoards map[string]boardTable - threads map[int]int // old thread id (previously stored in posts as the id) to new thread id (threads.id) + boards map[string]migrationBoard + threads map[int]gcsql.Thread // old thread id (previously stored in posts as the id) to new thread id (threads.id) } // IsMigratingInPlace implements common.DBMigrator. diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index b0c49809..fe0ae120 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -3,8 +3,8 @@ package pre2021 const ( sectionsQuery = `SELECT id, list_order, hidden, name, abbreviation FROM DBPREFIXsections` - boardsQuery = `SELECT id, list_order, dir, type, upload_type, title, subtitle, description, section, max_file_size, max_pages, -default_style, locked, created_on, anonymous, forced_anon, max_age, autosage_after, no_images_after, max_message_length, embeds_allowed, + boardsQuery = `SELECT id, list_order, dir, title, subtitle, description, section, max_file_size, max_pages, +default_style, locked, created_on, anonymous, forced_anon, autosage_after, no_images_after, max_message_length, embeds_allowed, redirect_to_thread, require_file, enable_catalog FROM DBPREFIXboards` From b34a956baa1cfb9083c0edb86d476dbde1cead49 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 1 Jan 2025 14:52:43 -0800 Subject: [PATCH 031/122] Start adding pre-2021 testing --- .../internal/pre2021/sqlite3_test.go | 54 ++++++++++++++++++ tools/gochan-pre2021.sqlite3db | Bin 0 -> 81920 bytes 2 files changed, 54 insertions(+) create mode 100644 cmd/gochan-migration/internal/pre2021/sqlite3_test.go create mode 100644 tools/gochan-pre2021.sqlite3db diff --git a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go new file mode 100644 index 00000000..0b6c1dfe --- /dev/null +++ b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go @@ -0,0 +1,54 @@ +package pre2021 + +import ( + "path" + "testing" + + "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" + "github.com/gochan-org/gochan/pkg/config" + "github.com/gochan-org/gochan/pkg/gcsql" + "github.com/gochan-org/gochan/pkg/gcutil/testutil" + "github.com/stretchr/testify/assert" +) + +const ( + sqlite3DBPath = "tools/gochan-pre2021.sqlite3db" // relative to gochan project root +) + +func TestMigrateToNewDB(t *testing.T) { + dir, err := testutil.GoToGochanRoot(t) + if !assert.NoError(t, err) { + return + } + if !assert.NoError(t, common.InitTestMigrationLog(t)) { + return + } + dbPath := path.Join(dir, sqlite3DBPath) + + oldSQLConfig := config.SQLConfig{ + DBtype: "sqlite3", + DBname: path.Base(dbPath), + DBhost: dbPath, + DBprefix: "gc_", + DBusername: "gochan", + DBpassword: "password", + } + migrator := Pre2021Migrator{ + config: Pre2021Config{ + SQLConfig: oldSQLConfig, + }, + } + outDir := t.TempDir() + + config.SetTestDBConfig("sqlite3", path.Join(outDir, "gochan-migrated.sqlite3db"), "gochan-migrated.sqlite3db", "gochan", "password", "gc_") + sqlConfig := config.GetSQLConfig() + + if !assert.NoError(t, gcsql.ConnectToDB(&sqlConfig)) { + return + } + if !assert.NoError(t, gcsql.CheckAndInitializeDatabase("sqlite3", "4")) { + return + } + + assert.NoError(t, migrator.migrateBoardsToNewDB()) +} diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db new file mode 100644 index 0000000000000000000000000000000000000000..137996f869215ea9a1c309704733aa93078447df GIT binary patch literal 81920 zcmeI5&2QW09mh%0vR)F2NgPA)QqMHf%$5@?da<3vE!L`@+5~l+)Uk^dL#xpeZLy(9 zg{0zSKo3saVZe6ScG-P@zvkAAV2ADZ@XeA)+ldei zjq#;riF}^l^YZ&V?++#Z?!2+8+KRMom~GjXD%>fK=ed_9iR0*$Ei$a5C8!X009sH0T2KI5C8!X009s zc_v^f4O=yIE5_FUG4U6i_$j^N009sH0T2KI5C8!X009sH0T2KI5cmoZn2jYke!7wz zi&dT70;!`}-{f6;Kmd009sH0T2KI5C8!X009sHf#XJCPl%u4Qs*+sOh(LADk+)IRFYQ< z&flEeY^(af;%;fVSd4IbwX11TN1;c!(M@B?&?T9!-CM%aY*%-efby26AI>ZFxtLC=;;=&pn=}gfc9l4Ab@+UnP|X zwJa?!E)7KPIXPN-3Nl2+Lh*8GNl^9svZgjKNmk3~YE5ZNk<=aCFqLL+9=tgAF2pDg z7bfX7PI*Ya-K*^4Qla$Dn%q_jbI;S=b46#TJL-0$r8MqZ-L|dJJ@knDURy4fDXk@c z6;5d}ryI8=!<0;AM|mKzsMwVsEj#z4?JE;6?6%NXCex|DGATbvnb<8;kqL`DSeaOX zanAk!>|Z$X9=+iJ0T2KI5C8!X009sH0T2KI5CDN^kHFqoY=TQpB$F5BWnSQ?l9j1s zGM0>`Qpr?HZ8jDCy7TVtl)S>eLC>CpFOi?B?`*VG%k}BqSkNWAr7#H!EqPy}k6q`u z6CJgqScQhwcRRkcyii;&UaRVc{;+LyE$(*`w|A1KT#YA_GnZdWIk}n-a>X_nHt3&{ zAIucw`4 z$tnQ-T7Vw#P%aeN_y2kEUmU$~fB*=900@8p2!H?xfB*=900@8p2plH@Y2jST+XcY) z|Bq9_LKhGK0T2KI5C8!X009sH0T2KI5Qrwg*8f8G9w$oKKhXyc5C8!X009sH0T2KI z5C8!X009tqIt0EeJje0O1Yl-nW{lpG?8uPaL#{$6G4GN5uF<-_i_Y$VLBEAXvUcBZ z$?@^arI%hBo4hg^ z3Js%u$5tD6Ri#Px0vz*uw{ANI1<0m= zEYXwvd;JddpiGv^m4(8)ogLq9VBIj)9aWb#=I9#`9Qor}I((Cj%!=V|OST7;DKA&9 zML14To=%Y?77zdd5C8#(zz@`TA~!X~ z|76-`HhFg%b=OpHa5B2GUaM}@q>bt;t2HTbmS)o_wJF_L+o-)(TbFLH-?&*_e@lA3 z_LfxL+_-gPje^~*t!+qax9ER!b@fs@rKy%pV@6Xkr8_sPtE&_^@Rq(+d$qc`x*-)w zu&J8To7MG|>(%wyg~hzIxpw1?&Dy@~%JtgH>(VR}d-0pn%uJrb*bh6(5z%%#njtsq zk0EKRwx;xD!B<1_(sm_+)UEDTf0(6({C<9Y-DYN(a{D%Nf)oyx_RRV)(Bb$>6kS_> zP-g>j-BQ1&M2jt4DP3K>R`!HDWEI&ugj9*e+EljXu4dOQ`{6+#`jpa)#$B3UymIT- zYOT5!&grYw)jKtcw?Xp{TWQwGEcC{Wo3%R|)tk4&(kq+m>r{8@fe3|^eY-^gG*&Z$ zVhM5+*cD=|$%0aYQe$C{&Ko1iEJjD7X4O*e+J;3Eb$Oc%7DdaIi%e2C>W=kk>o8F! z$kJ>pmJ=)4c;2yF(a~O|2^EPe?JcEAk!qT;dl>GG^-V^b%ucJO(y;5cQTL{P2bNF} z#HwUS1n4Andcr%(QiBc zv^z-nkQkjF!Jm>pP5`&pKT2|r<(BM@Z*K>Zi^Zr`$mDF5YPL;_ke;UQQ;cfi%7YHo zJgY}K)ViaB1*YbGZlO>?)s`*l2Ogt5ju;^-K7_h%*!#o8A;sYuy|2smlPcGc^|~84 zs1_mhY_qGR^N%hHP7C=U7|7jl7PS!XY-F&7c<#fr5NdS&J(V{uGItski)={tIyPye z_U)d*%MC zQ*9bRm3v)9ZzuzoLLapO&so;QFHy++qbi@s&Cl~6Ec!i^tm{UX;%h6q?UX|Ab*wK| zRz|(P!;}b3LEfdlKB(W(!y{{$KKjQimyW^A^F>GNm8BCwhS8C+(1dL>|4m-?xOOt7_E;nrTJ`J8vuvAOuFP|FCtxWQq zsyCGf-gwEDPR@1R$6hb_XJb^IldMkf$=O8i{CWQUV3Kdq(ukf8!z#3YdOB2Gp3E>M z=9HGR%}`p}uVn<-ORY%@o1n_}4yYtD%yc!K-!n6=ju(T*5&_4sFeluAy4(w>9#Ux3)~>zA7^%0#>D-icBp~KIzO&Kg{?_n@Yzpomr80CJw6X zxeueU(>j8z3=|uitGF9i!R7%e5jFYrbquzqLNgW$c)&W4d}#G};Bte!8yfYIJ@eO4 zYW_r)7P$(oq5OGnv|Kv>gM|}LFZbgYe8x;u)^eQBlY@L6*P-c#33f;ueUNN`Zl>t3 zbAxi}T{yB5pc_{Wqq)DF>a9ZL@7&ZnDDdFiw^Zz@e-t=!#=GxcH|5>_6$1JB-E@%V zKt&2I$9w(v6=!<;qQMQbAkIV9jL}Q@!xK4p50Gs(Ia^Zo-SCJKbAFz zjLVfeDy{?CP>?HUT42jN0kz&4N8j9G=w|G1?H$p~2x3#w6t;sIkrerEb=w_k)K4;C zQmEh_?BR`^Bp-L5mG#KVp<9IBLTkrs{WDx8Z!A80m9|*j_ zC%N1NFIFmYvrV=Ewi;wLK$z?QML>m&8C;EUN>c`T@k@?gI6wddKmY_l00ck)1V8`; zKmY_l00fQ~fuxX2IlBe!@BcsH#D9vP94{524G4e$2!H?xfB*=900@8p2!H?x95(`a z`oaH9X*~5#VQ$v3(3LODJx?cHO*SR6nnX4knNcU3jDu$A0RZEt#pJeWv~6SVe}r16 AjsO4v literal 0 HcmV?d00001 From 90986e58b8481092c223de573221071165776f7f Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 1 Jan 2025 14:53:10 -0800 Subject: [PATCH 032/122] Add pre2021 section migration --- .../internal/common/logger.go | 11 +++ .../internal/pre2021/boards.go | 73 ++++++++++++------- .../internal/pre2021/pre2021.go | 3 +- 3 files changed, 59 insertions(+), 28 deletions(-) diff --git a/cmd/gochan-migration/internal/common/logger.go b/cmd/gochan-migration/internal/common/logger.go index 837d040b..0048d765 100644 --- a/cmd/gochan-migration/internal/common/logger.go +++ b/cmd/gochan-migration/internal/common/logger.go @@ -5,6 +5,7 @@ import ( "io/fs" "os" "path" + "testing" "github.com/gochan-org/gochan/pkg/config" "github.com/gochan-org/gochan/pkg/gcutil" @@ -21,6 +22,16 @@ var ( migrationLog zerolog.Logger ) +func InitTestMigrationLog(t *testing.T) (err error) { + dir := os.TempDir() + migrationLogFile, err = os.CreateTemp(dir, "migration-test") + if err != nil { + return err + } + migrationLog = zerolog.New(zerolog.NewTestWriter(t)) + return nil +} + func InitMigrationLog() (err error) { if migrationLogFile != nil { // Migration log already initialized diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index da483068..0678e9a4 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -1,7 +1,6 @@ package pre2021 import ( - "errors" "runtime/debug" "strings" @@ -12,25 +11,30 @@ import ( ) type migrationBoard struct { - oldID int + oldSectionID int gcsql.Board } -func (m *Pre2021Migrator) migrateBoardsInPlace() error { - return nil +type migrationSection struct { + oldID int + gcsql.Section } -func (m *Pre2021Migrator) createSectionIfNotExist(sectionCheck *gcsql.Section) (int, error) { - // to be used when not migrating in place, otherwise the section table should be altered - section, err := gcsql.GetSectionFromName(sectionCheck.Name) - if errors.Is(err, gcsql.ErrSectionDoesNotExist) { - // section doesn't exist, create it - section, err = gcsql.NewSection(sectionCheck.Name, sectionCheck.Abbreviation, true, 0) - if err != nil { - return 0, err - } +func (m *Pre2021Migrator) migrateSectionsInPlace() error { + return common.NewMigrationError("pre2021", "migrateSectionsInPlace not implemented") +} + +func (m *Pre2021Migrator) migrateBoardsInPlace() error { + errEv := common.LogError() + defer errEv.Discard() + err := m.migrateSectionsInPlace() + if err != nil { + errEv.Err(err).Caller().Msg("Failed to migrate sections") + return err } - return section.ID, nil + err = common.NewMigrationError("pre2021", "migrateBoardsInPlace not implemented") + errEv.Err(err).Caller().Msg("Failed to migrate boards") + return err } func (m *Pre2021Migrator) migrateSectionsToNewDB() error { @@ -41,7 +45,8 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error { return err } defer rows.Close() - + errEv := common.LogError() + defer errEv.Discard() for rows.Next() { var section gcsql.Section if err = rows.Scan( @@ -53,19 +58,29 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error { ); err != nil { return err } - if _, err = m.createSectionIfNotExist(§ion); err != nil { + m.sections = append(m.sections, migrationSection{ + oldID: section.ID, + Section: section, + }) + + for _, newSection := range gcsql.AllSections { + if newSection.Name == section.Name || newSection.Abbreviation == section.Abbreviation { + common.LogWarning().Str("section", section.Name).Msg("Section already exists in new db, moving on") + m.sections[len(m.sections)-1].ID = newSection.ID + break + } + } + if _, err = gcsql.NewSection(section.Name, section.Abbreviation, false, section.Position); err != nil { + errEv.Err(err).Caller(). + Str("sectionName", section.Name). + Msg("Failed to create section") return err } } if err = rows.Close(); err != nil { + errEv.Caller().Msg("Failed to close section rows") return err } - m.migrationSectionID, err = m.createSectionIfNotExist(&gcsql.Section{ - Name: "Migrated Boards", - Abbreviation: "mb", - Hidden: true, - }) - return err } @@ -76,14 +91,16 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { errEv := common.LogError() defer errEv.Discard() - err := m.migrateSectionsToNewDB() + // get all boards from new db + err := gcsql.ResetBoardSectionArrays() if err != nil { - errEv.Err(err).Msg("Failed to migrate sections") + errEv.Err(err).Caller().Msg("Failed to reset board section arrays") + return nil } - // get all boards from new db - if err = gcsql.ResetBoardSectionArrays(); err != nil { - return nil + if err = m.migrateSectionsToNewDB(); err != nil { + errEv.Err(err).Caller().Msg("Failed to migrate sections") + return err } // get boards from old db @@ -117,10 +134,12 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { break } } + m.boards[board.Dir] = board if found { continue } + // create new board using the board data from the old db // omitting things like ID and creation date since we don't really care if err = gcsql.CreateBoard(&board.Board, false); err != nil { diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index 3d04f99c..47c80762 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -25,7 +25,8 @@ type Pre2021Migrator struct { migrationSectionID int posts []postTable boards map[string]migrationBoard - threads map[int]gcsql.Thread // old thread id (previously stored in posts as the id) to new thread id (threads.id) + sections []migrationSection + threads map[int]gcsql.Thread // old thread id (previously stored in posts ) to new thread id (threads.id) } // IsMigratingInPlace implements common.DBMigrator. From beb048716eb3acadd66db5019de5dcf1e94b6bfa Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 1 Jan 2025 17:58:09 -0800 Subject: [PATCH 033/122] Move setup functionality in test to a separate function --- .../internal/pre2021/sqlite3_test.go | 61 +++++++++++++------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go index 0b6c1dfe..a93c1a1f 100644 --- a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go +++ b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go @@ -4,7 +4,6 @@ import ( "path" "testing" - "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/config" "github.com/gochan-org/gochan/pkg/gcsql" "github.com/gochan-org/gochan/pkg/gcutil/testutil" @@ -15,40 +14,68 @@ const ( sqlite3DBPath = "tools/gochan-pre2021.sqlite3db" // relative to gochan project root ) -func TestMigrateToNewDB(t *testing.T) { +func setupMigrationTest(t *testing.T) *Pre2021Migrator { dir, err := testutil.GoToGochanRoot(t) if !assert.NoError(t, err) { - return - } - if !assert.NoError(t, common.InitTestMigrationLog(t)) { - return + t.FailNow() } dbPath := path.Join(dir, sqlite3DBPath) oldSQLConfig := config.SQLConfig{ - DBtype: "sqlite3", - DBname: path.Base(dbPath), - DBhost: dbPath, - DBprefix: "gc_", - DBusername: "gochan", - DBpassword: "password", + DBtype: "sqlite3", + DBname: path.Base(dbPath), + DBhost: dbPath, + DBprefix: "gc_", + DBusername: "gochan", + DBpassword: "password", + DBTimeoutSeconds: 600, } - migrator := Pre2021Migrator{ + migrator := &Pre2021Migrator{ config: Pre2021Config{ SQLConfig: oldSQLConfig, }, } - outDir := t.TempDir() + db, err := gcsql.Open(&oldSQLConfig) + if !assert.NoError(t, err) { + t.FailNow() + } + migrator.db = db + outDir := t.TempDir() config.SetTestDBConfig("sqlite3", path.Join(outDir, "gochan-migrated.sqlite3db"), "gochan-migrated.sqlite3db", "gochan", "password", "gc_") sqlConfig := config.GetSQLConfig() + sqlConfig.DBTimeoutSeconds = 600 if !assert.NoError(t, gcsql.ConnectToDB(&sqlConfig)) { - return + t.FailNow() } if !assert.NoError(t, gcsql.CheckAndInitializeDatabase("sqlite3", "4")) { - return + t.FailNow() } - assert.NoError(t, migrator.migrateBoardsToNewDB()) + return migrator +} + +func TestMigrateToNewDB(t *testing.T) { + migrator := setupMigrationTest(t) + + assert.NoError(t, migrator.migrateBoardsToNewDB()) + + newBoards, err := gcsql.GetAllBoards(false) + if !assert.NoError(t, err) { + return + } + assert.GreaterOrEqual(t, len(newBoards), 2, "Expected new boards list to have at least 2 boards") // old DB has 2 boards, /test/ and /hidden/ + + hiddenBoard, err := gcsql.GetBoardFromDir("hidden") + if !assert.NoError(t, err) { + return + } + t.Logf("Hidden board section ID: %d", hiddenBoard.SectionID) + hiddenSection, err := gcsql.GetSectionFromID(hiddenBoard.SectionID) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, "Hidden", hiddenSection.Name, "Expected Hidden section to have name 'Hidden'") + assert.True(t, hiddenSection.Hidden, "Expected Hidden section to be hidden") } From 525c5953d909c03722ea6d8322be38a78578e700 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Thu, 2 Jan 2025 17:00:25 -0800 Subject: [PATCH 034/122] Properly migrate board and sections, and have board point to correct section --- .../internal/pre2021/boards.go | 73 ++++++++++++------ .../internal/pre2021/sqlite3_test.go | 42 +++++++--- tools/gochan-pre2021.sqlite3db | Bin 81920 -> 81920 bytes 3 files changed, 80 insertions(+), 35 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index 0678e9a4..1dd2aede 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -40,48 +40,59 @@ func (m *Pre2021Migrator) migrateBoardsInPlace() error { func (m *Pre2021Migrator) migrateSectionsToNewDB() error { // creates sections in the new db if they don't exist, and also creates a migration section that // boards will be set to, to be moved to the correct section by the admin after migration + errEv := common.LogError() + defer errEv.Discard() + // populate m.sections with all sections from the new db + currentAllSections, err := gcsql.GetAllSections(false) + if err != nil { + errEv.Err(err).Caller().Msg("Failed to get all sections from new db") + return err + } + + for _, section := range currentAllSections { + m.sections = append(m.sections, migrationSection{ + oldID: -1, + Section: section, + }) + } + rows, err := m.db.QuerySQL(sectionsQuery) if err != nil { + errEv.Err(err).Caller().Msg("Failed to query old database sections") return err } defer rows.Close() - errEv := common.LogError() - defer errEv.Discard() for rows.Next() { var section gcsql.Section - if err = rows.Scan( - §ion.ID, - §ion.Position, - §ion.Hidden, - §ion.Name, - §ion.Abbreviation, - ); err != nil { + if err = rows.Scan(§ion.ID, §ion.Position, §ion.Hidden, §ion.Name, §ion.Abbreviation); err != nil { + errEv.Err(err).Caller().Msg("Failed to scan row into section") return err } - m.sections = append(m.sections, migrationSection{ - oldID: section.ID, - Section: section, - }) + var found bool + for s, newSection := range m.sections { + if section.Name == newSection.Name { + m.sections[s].oldID = section.ID - for _, newSection := range gcsql.AllSections { - if newSection.Name == section.Name || newSection.Abbreviation == section.Abbreviation { - common.LogWarning().Str("section", section.Name).Msg("Section already exists in new db, moving on") - m.sections[len(m.sections)-1].ID = newSection.ID + found = true break } } - if _, err = gcsql.NewSection(section.Name, section.Abbreviation, false, section.Position); err != nil { - errEv.Err(err).Caller(). - Str("sectionName", section.Name). - Msg("Failed to create section") - return err + if !found { + migratedSection, err := gcsql.NewSection(section.Name, section.Abbreviation, section.Hidden, section.Position) + if err != nil { + errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to migrate section") + return err + } + m.sections = append(m.sections, migrationSection{ + Section: *migratedSection, + }) } } if err = rows.Close(); err != nil { errEv.Caller().Msg("Failed to close section rows") return err } - return err + return nil } func (m *Pre2021Migrator) migrateBoardsToNewDB() error { @@ -99,7 +110,7 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { } if err = m.migrateSectionsToNewDB(); err != nil { - errEv.Err(err).Caller().Msg("Failed to migrate sections") + // error already logged return err } @@ -110,6 +121,7 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { return err } defer rows.Close() + for rows.Next() { var board migrationBoard var maxPages int @@ -134,9 +146,18 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { break } } + if !found { + for _, section := range m.sections { + if section.oldID == board.oldSectionID { + board.SectionID = section.ID + break + } + } + } m.boards[board.Dir] = board if found { + // TODO: update board title, subtitle, section etc. in new db continue } @@ -148,6 +169,10 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { } common.LogInfo().Str("board", board.Dir).Msg("Board successfully created") } + if err = gcsql.ResetBoardSectionArrays(); err != nil { + errEv.Err(err).Caller().Msg("Failed to reset board and section arrays") + return err + } return nil } diff --git a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go index a93c1a1f..c0e0ad95 100644 --- a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go +++ b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go @@ -14,7 +14,7 @@ const ( sqlite3DBPath = "tools/gochan-pre2021.sqlite3db" // relative to gochan project root ) -func setupMigrationTest(t *testing.T) *Pre2021Migrator { +func setupMigrationTest(t *testing.T, outDir string) *Pre2021Migrator { dir, err := testutil.GoToGochanRoot(t) if !assert.NoError(t, err) { t.FailNow() @@ -40,9 +40,10 @@ func setupMigrationTest(t *testing.T) *Pre2021Migrator { t.FailNow() } migrator.db = db + migratedDBPath := path.Join(outDir, "gochan-migrated.sqlite3db") + t.Log("Migrated DB path:", migratedDBPath) - outDir := t.TempDir() - config.SetTestDBConfig("sqlite3", path.Join(outDir, "gochan-migrated.sqlite3db"), "gochan-migrated.sqlite3db", "gochan", "password", "gc_") + config.SetTestDBConfig("sqlite3", migratedDBPath, path.Base(migratedDBPath), "gochan", "password", "gc_") sqlConfig := config.GetSQLConfig() sqlConfig.DBTimeoutSeconds = 600 @@ -56,26 +57,45 @@ func setupMigrationTest(t *testing.T) *Pre2021Migrator { return migrator } -func TestMigrateToNewDB(t *testing.T) { - migrator := setupMigrationTest(t) +func TestMigrateBoardsToNewDB(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir) + assert.NoError(t, gcsql.ResetBoardSectionArrays()) + + numBoards := len(gcsql.AllBoards) + numSections := len(gcsql.AllSections) + + assert.Equal(t, 1, numBoards, "Expected to have 1 board pre-migration (/test/ is automatically created during provisioning)") + assert.Equal(t, 1, numSections, "Expected to have 1 section pre-migration (Main is automatically created during provisioning)") assert.NoError(t, migrator.migrateBoardsToNewDB()) - newBoards, err := gcsql.GetAllBoards(false) + migratedBoards, err := gcsql.GetAllBoards(false) if !assert.NoError(t, err) { - return + t.FailNow() } - assert.GreaterOrEqual(t, len(newBoards), 2, "Expected new boards list to have at least 2 boards") // old DB has 2 boards, /test/ and /hidden/ + migratedSections, err := gcsql.GetAllSections(false) + if !assert.NoError(t, err) { + t.FailNow() + } + + assert.Equal(t, len(migratedBoards), 2, "Expected updated boards list to have two boards") + assert.Equal(t, len(migratedSections), 2, "Expected updated sections list to have two sections") hiddenBoard, err := gcsql.GetBoardFromDir("hidden") if !assert.NoError(t, err) { - return + t.FailNow() } t.Logf("Hidden board section ID: %d", hiddenBoard.SectionID) + + t.Log("Number of sections:", len(migratedSections)) + for _, section := range migratedSections { + t.Logf("Section ID %d: %#v", section.ID, section) + } hiddenSection, err := gcsql.GetSectionFromID(hiddenBoard.SectionID) if !assert.NoError(t, err) { - return + t.FailNow() } - assert.Equal(t, "Hidden", hiddenSection.Name, "Expected Hidden section to have name 'Hidden'") + assert.Equal(t, "Hidden section", hiddenSection.Name, "Expected /hidden/ board's section to have name 'Hidden'") assert.True(t, hiddenSection.Hidden, "Expected Hidden section to be hidden") } diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db index 137996f869215ea9a1c309704733aa93078447df..c308c532ce9db711bef1f352a00e607e8095c9ee 100644 GIT binary patch delta 218 zcmZo@U~On%ogmF9F;T{uQDS4lLVs@lLIx&2H3mL4zS|ocrTBOnjTj{vI7B%()I)P8 z*Tx%}D-;x^>KYjs87kyvrWYlaWaj5NOvN3RQh^qT0X6EIBFw6=mCN>66 fPEARV%#@VWJcZ)aWRS%fU>?v!0U(BhC5#RLU(-F} delta 146 zcmZo@U~On%ogmF9Fj2;tQD9@jLVs=s1_ma++YJ1Le783?p68oXIIrDXDo1#i_|9nfZAcnJGXXA0rzB2ZyMrZ(?R%E)XwabN~P Date: Thu, 2 Jan 2025 19:51:38 -0800 Subject: [PATCH 035/122] Fully migrate and update board and section data to new DB for pre-2021 --- .../internal/pre2021/boards.go | 78 ++++++++++++------ .../internal/pre2021/pre2021.go | 8 +- .../internal/pre2021/sqlite3_test.go | 40 +++++---- tools/gochan-pre2021.sqlite3db | Bin 81920 -> 81920 bytes 4 files changed, 80 insertions(+), 46 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index 1dd2aede..26b7d7f6 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -12,6 +12,7 @@ import ( type migrationBoard struct { oldSectionID int + oldID int gcsql.Board } @@ -71,8 +72,20 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error { var found bool for s, newSection := range m.sections { if section.Name == newSection.Name { + // section already exists, update values m.sections[s].oldID = section.ID - + m.sections[s].Abbreviation = section.Abbreviation + m.sections[s].Hidden = section.Hidden + m.sections[s].Position = section.Position + common.LogInfo(). + Int("sectionID", section.ID). + Int("oldSectionID", m.sections[s].oldID). + Str("sectionName", section.Name). + Str("sectionAbbreviation", section.Abbreviation). + Msg("Section already exists in new db, updating values") + if err = m.sections[s].UpdateValues(); err != nil { + errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to update pre-existing section values") + } found = true break } @@ -96,9 +109,7 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error { } func (m *Pre2021Migrator) migrateBoardsToNewDB() error { - if m.boards == nil { - m.boards = make(map[string]migrationBoard) - } + m.boards = nil errEv := common.LogError() defer errEv.Discard() @@ -110,10 +121,22 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { } if err = m.migrateSectionsToNewDB(); err != nil { - // error already logged + // error should already be logged by migrateSectionsToNewDB return err } + allBoards, err := gcsql.GetAllBoards(false) + if err != nil { + errEv.Err(err).Caller().Msg("Failed to get all boards from new db") + return err + } + for _, board := range allBoards { + m.boards = append(m.boards, migrationBoard{ + oldSectionID: -1, + Board: board, + }) + } + // get boards from old db rows, err := m.db.QuerySQL(boardsQuery) if err != nil { @@ -126,44 +149,49 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { var board migrationBoard var maxPages int if err = rows.Scan( - &board.ID, &board.NavbarPosition, &board.Dir, &board.Title, &board.Subtitle, - &board.Description, &board.SectionID, &board.MaxFilesize, &maxPages, &board.DefaultStyle, &board.Locked, - &board.CreatedAt, &board.AnonymousName, &board.ForceAnonymous, &board.AutosageAfter, &board.NoImagesAfter, - &board.MaxMessageLength, &board.AllowEmbeds, &board.RedirectToThread, &board.RequireFile, &board.EnableCatalog, + &board.ID, &board.NavbarPosition, &board.Dir, &board.Title, &board.Subtitle, &board.Description, + &board.SectionID, &board.MaxFilesize, &maxPages, &board.DefaultStyle, &board.Locked, &board.CreatedAt, + &board.AnonymousName, &board.ForceAnonymous, &board.AutosageAfter, &board.NoImagesAfter, &board.MaxMessageLength, + &board.AllowEmbeds, &board.RedirectToThread, &board.RequireFile, &board.EnableCatalog, ); err != nil { errEv.Err(err).Caller().Msg("Failed to scan row into board") return err } board.MaxThreads = maxPages * config.GetBoardConfig(board.Dir).ThreadsPerPage found := false - for _, newBoard := range gcsql.AllBoards { - if _, ok := m.boards[board.Dir]; !ok { - m.boards[board.Dir] = board - } + + for b, newBoard := range m.boards { if newBoard.Dir == board.Dir { - common.LogWarning().Str("board", board.Dir).Msg("Board already exists in new db, moving on") + m.boards[b].oldID = board.ID + m.boards[b].oldSectionID = board.SectionID + common.LogInfo().Str("board", board.Dir).Msg("Board already exists in new db, updating values") + // don't update other values in the array since they don't affect migrating threads or posts + if _, err = gcsql.ExecSQL(`UPDATE DBPREFIXboards + SET uri = ?, navbar_position = ?, title = ?, subtitle = ?, description = ?, + max_file_size = ?, max_threads = ?, default_style = ?, locked = ?, + anonymous_name = ?, force_anonymous = ?, autosage_after = ?, no_images_after = ?, max_message_length = ?, + min_message_length = ?, allow_embeds = ?, redirect_to_thread = ?, require_file = ?, enable_catalog = ? + WHERE id = ?`, + board.Dir, board.NavbarPosition, board.Title, board.Subtitle, board.Description, + board.MaxFilesize, board.MaxThreads, board.DefaultStyle, board.Locked, + board.AnonymousName, board.ForceAnonymous, board.AutosageAfter, board.NoImagesAfter, board.MaxMessageLength, + board.MinMessageLength, board.AllowEmbeds, board.RedirectToThread, board.RequireFile, board.EnableCatalog, + newBoard.ID); err != nil { + errEv.Err(err).Caller().Str("board", board.Dir).Msg("Failed to update board values") + return err + } found = true break } } - if !found { - for _, section := range m.sections { - if section.oldID == board.oldSectionID { - board.SectionID = section.ID - break - } - } - } - m.boards[board.Dir] = board if found { - // TODO: update board title, subtitle, section etc. in new db continue } // create new board using the board data from the old db // omitting things like ID and creation date since we don't really care - if err = gcsql.CreateBoard(&board.Board, false); err != nil { + if err = gcsql.CreateBoard(&board.Board, board.IsHidden(false)); err != nil { errEv.Err(err).Caller().Str("board", board.Dir).Msg("Failed to create board") return err } diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index 47c80762..8892d75e 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -22,11 +22,9 @@ type Pre2021Migrator struct { options *common.MigrationOptions config Pre2021Config - migrationSectionID int - posts []postTable - boards map[string]migrationBoard - sections []migrationSection - threads map[int]gcsql.Thread // old thread id (previously stored in posts ) to new thread id (threads.id) + posts []postTable + boards []migrationBoard + sections []migrationSection } // IsMigratingInPlace implements common.DBMigrator. diff --git a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go index c0e0ad95..868c123c 100644 --- a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go +++ b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go @@ -14,7 +14,7 @@ const ( sqlite3DBPath = "tools/gochan-pre2021.sqlite3db" // relative to gochan project root ) -func setupMigrationTest(t *testing.T, outDir string) *Pre2021Migrator { +func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre2021Migrator { dir, err := testutil.GoToGochanRoot(t) if !assert.NoError(t, err) { t.FailNow() @@ -41,7 +41,6 @@ func setupMigrationTest(t *testing.T, outDir string) *Pre2021Migrator { } migrator.db = db migratedDBPath := path.Join(outDir, "gochan-migrated.sqlite3db") - t.Log("Migrated DB path:", migratedDBPath) config.SetTestDBConfig("sqlite3", migratedDBPath, path.Base(migratedDBPath), "gochan", "password", "gc_") sqlConfig := config.GetSQLConfig() @@ -59,7 +58,7 @@ func setupMigrationTest(t *testing.T, outDir string) *Pre2021Migrator { func TestMigrateBoardsToNewDB(t *testing.T) { outDir := t.TempDir() - migrator := setupMigrationTest(t, outDir) + migrator := setupMigrationTest(t, outDir, false) assert.NoError(t, gcsql.ResetBoardSectionArrays()) numBoards := len(gcsql.AllBoards) @@ -68,7 +67,7 @@ func TestMigrateBoardsToNewDB(t *testing.T) { assert.Equal(t, 1, numBoards, "Expected to have 1 board pre-migration (/test/ is automatically created during provisioning)") assert.Equal(t, 1, numSections, "Expected to have 1 section pre-migration (Main is automatically created during provisioning)") - assert.NoError(t, migrator.migrateBoardsToNewDB()) + assert.NoError(t, migrator.MigrateBoards()) migratedBoards, err := gcsql.GetAllBoards(false) if !assert.NoError(t, err) { @@ -82,20 +81,29 @@ func TestMigrateBoardsToNewDB(t *testing.T) { assert.Equal(t, len(migratedBoards), 2, "Expected updated boards list to have two boards") assert.Equal(t, len(migratedSections), 2, "Expected updated sections list to have two sections") + // Test migrated sections + mainSection, err := gcsql.GetSectionFromName("Main") + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, "mainmigration", mainSection.Abbreviation, "Expected Main section to have updated abbreviation name 'mainmigration'") + + // Test migrated boards + testBoard, err := gcsql.GetBoardFromDir("test") + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, "Testing Board", testBoard.Title) + assert.Equal(t, "Board for testing pre-2021 migration", testBoard.Subtitle) + testBoardSection, err := gcsql.GetSectionFromID(testBoard.SectionID) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, "Main", testBoardSection.Name, "Expected /test/ board to be in Main section") + hiddenBoard, err := gcsql.GetBoardFromDir("hidden") if !assert.NoError(t, err) { t.FailNow() } - t.Logf("Hidden board section ID: %d", hiddenBoard.SectionID) - - t.Log("Number of sections:", len(migratedSections)) - for _, section := range migratedSections { - t.Logf("Section ID %d: %#v", section.ID, section) - } - hiddenSection, err := gcsql.GetSectionFromID(hiddenBoard.SectionID) - if !assert.NoError(t, err) { - t.FailNow() - } - assert.Equal(t, "Hidden section", hiddenSection.Name, "Expected /hidden/ board's section to have name 'Hidden'") - assert.True(t, hiddenSection.Hidden, "Expected Hidden section to be hidden") + assert.Equal(t, "Hidden Board", hiddenBoard.Title) } diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db index c308c532ce9db711bef1f352a00e607e8095c9ee..632d3be5712b5d68cd31bb3a3f185a2a330716cc 100644 GIT binary patch delta 32 ocmZo@U~On%ogmF9IZ?)$QF3F#yI4l2$^LO$8J#w>#{Yf*0ICiPqW}N^ delta 32 ocmZo@U~On%ogmF9F;T{uQDS4lyI97g$^LO$8Iv}%#{Yf*0IVeo;Q#;t From 4699417370739e3235da57a090d2728bb91c6eac Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Thu, 2 Jan 2025 20:03:26 -0800 Subject: [PATCH 036/122] Start adding stuff for testing migrating pre-2021 DB in place --- .../internal/pre2021/sqlite3_test.go | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go index 868c123c..28540fe1 100644 --- a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go +++ b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go @@ -1,6 +1,8 @@ package pre2021 import ( + "io" + "os" "path" "testing" @@ -20,6 +22,27 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 t.FailNow() } dbPath := path.Join(dir, sqlite3DBPath) + if migrateInPlace { + oldDbFile, err := os.Open(dbPath) + if !assert.NoError(t, err) { + t.FailNow() + } + defer oldDbFile.Close() + + newDbFile, err := os.OpenFile(path.Join(outDir, "gochan-pre2021.sqlite3db"), os.O_CREATE|os.O_RDWR, 0644) + if !assert.NoError(t, err) { + t.FailNow() + } + defer newDbFile.Close() + + _, err = io.Copy(newDbFile, oldDbFile) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.NoError(t, oldDbFile.Close()) + assert.NoError(t, newDbFile.Close()) + dbPath = path.Join(outDir, "gochan-pre2021.sqlite3db") + } oldSQLConfig := config.SQLConfig{ DBtype: "sqlite3", @@ -59,6 +82,9 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 func TestMigrateBoardsToNewDB(t *testing.T) { outDir := t.TempDir() migrator := setupMigrationTest(t, outDir, false) + if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { + t.FailNow() + } assert.NoError(t, gcsql.ResetBoardSectionArrays()) numBoards := len(gcsql.AllBoards) @@ -67,7 +93,9 @@ func TestMigrateBoardsToNewDB(t *testing.T) { assert.Equal(t, 1, numBoards, "Expected to have 1 board pre-migration (/test/ is automatically created during provisioning)") assert.Equal(t, 1, numSections, "Expected to have 1 section pre-migration (Main is automatically created during provisioning)") - assert.NoError(t, migrator.MigrateBoards()) + if !assert.NoError(t, migrator.MigrateBoards()) { + t.FailNow() + } migratedBoards, err := gcsql.GetAllBoards(false) if !assert.NoError(t, err) { From 00fbd8f6c3603584d5c4c29f447f835731789568 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Thu, 2 Jan 2025 20:31:55 -0800 Subject: [PATCH 037/122] Add TestMigrateBoardsInPlace --- .../internal/common/handler.go | 8 +++- .../internal/pre2021/sqlite3_test.go | 40 ++++++++++++++----- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/cmd/gochan-migration/internal/common/handler.go b/cmd/gochan-migration/internal/common/handler.go index fa3e64ee..b997f81d 100644 --- a/cmd/gochan-migration/internal/common/handler.go +++ b/cmd/gochan-migration/internal/common/handler.go @@ -25,10 +25,14 @@ func (me *MigrationError) OldChanType() string { func (me *MigrationError) Error() string { from := me.oldChanType + errStr := "unable to migrate" if from != "" { - from = " from " + from + errStr += " from " + from } - return "unable to migrate " + from + ": " + me.errMessage + if me.errMessage != "" { + errStr += ": " + me.errMessage + } + return errStr } func NewMigrationError(oldChanType string, errMessage string) *MigrationError { diff --git a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go index 28540fe1..49bed7a1 100644 --- a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go +++ b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go @@ -13,7 +13,7 @@ import ( ) const ( - sqlite3DBPath = "tools/gochan-pre2021.sqlite3db" // relative to gochan project root + sqlite3DBDir = "tools/" // relative to gochan project root ) func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre2021Migrator { @@ -21,15 +21,19 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 if !assert.NoError(t, err) { t.FailNow() } - dbPath := path.Join(dir, sqlite3DBPath) + dbName := "gochan-pre2021.sqlite3db" + dbHost := path.Join(dir, sqlite3DBDir, dbName) + migratedDBName := "gochan-migrated.sqlite3db" + migratedDBHost := path.Join(outDir, migratedDBName) + if migrateInPlace { - oldDbFile, err := os.Open(dbPath) + oldDbFile, err := os.Open(dbHost) if !assert.NoError(t, err) { t.FailNow() } defer oldDbFile.Close() - newDbFile, err := os.OpenFile(path.Join(outDir, "gochan-pre2021.sqlite3db"), os.O_CREATE|os.O_RDWR, 0644) + newDbFile, err := os.OpenFile(migratedDBHost, os.O_CREATE|os.O_WRONLY, 0644) if !assert.NoError(t, err) { t.FailNow() } @@ -41,13 +45,14 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 } assert.NoError(t, oldDbFile.Close()) assert.NoError(t, newDbFile.Close()) - dbPath = path.Join(outDir, "gochan-pre2021.sqlite3db") + migratedDBHost = dbHost + migratedDBName = dbName } oldSQLConfig := config.SQLConfig{ DBtype: "sqlite3", - DBname: path.Base(dbPath), - DBhost: dbPath, + DBname: dbName, + DBhost: dbHost, DBprefix: "gc_", DBusername: "gochan", DBpassword: "password", @@ -63,17 +68,18 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 t.FailNow() } migrator.db = db - migratedDBPath := path.Join(outDir, "gochan-migrated.sqlite3db") - config.SetTestDBConfig("sqlite3", migratedDBPath, path.Base(migratedDBPath), "gochan", "password", "gc_") + config.SetTestDBConfig("sqlite3", migratedDBHost, migratedDBName, "gochan", "password", "gc_") sqlConfig := config.GetSQLConfig() sqlConfig.DBTimeoutSeconds = 600 if !assert.NoError(t, gcsql.ConnectToDB(&sqlConfig)) { t.FailNow() } - if !assert.NoError(t, gcsql.CheckAndInitializeDatabase("sqlite3", "4")) { - t.FailNow() + if !migrateInPlace { + if !assert.NoError(t, gcsql.CheckAndInitializeDatabase("sqlite3", "4")) { + t.FailNow() + } } return migrator @@ -135,3 +141,15 @@ func TestMigrateBoardsToNewDB(t *testing.T) { } assert.Equal(t, "Hidden Board", hiddenBoard.Title) } + +func TestMigrateBoardsInPlace(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, true) + if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { + t.FailNow() + } + + if !assert.Error(t, migrator.MigrateBoards(), "Not yet implemented") { + t.FailNow() + } +} From d19f0eebe3b4ebd121c6ee11d88427ee8e8ede6f Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Thu, 2 Jan 2025 20:36:19 -0800 Subject: [PATCH 038/122] Move board migration testing for pre2021 to a separate file --- .../internal/pre2021/boards_test.go | 77 +++++++++++++++++++ .../internal/pre2021/sqlite3_test.go | 69 ----------------- 2 files changed, 77 insertions(+), 69 deletions(-) create mode 100644 cmd/gochan-migration/internal/pre2021/boards_test.go diff --git a/cmd/gochan-migration/internal/pre2021/boards_test.go b/cmd/gochan-migration/internal/pre2021/boards_test.go new file mode 100644 index 00000000..f65b37d7 --- /dev/null +++ b/cmd/gochan-migration/internal/pre2021/boards_test.go @@ -0,0 +1,77 @@ +package pre2021 + +import ( + "testing" + + "github.com/gochan-org/gochan/pkg/gcsql" + "github.com/stretchr/testify/assert" +) + +func TestMigrateBoardsToNewDB(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, false) + if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { + t.FailNow() + } + assert.NoError(t, gcsql.ResetBoardSectionArrays()) + + numBoards := len(gcsql.AllBoards) + numSections := len(gcsql.AllSections) + + assert.Equal(t, 1, numBoards, "Expected to have 1 board pre-migration (/test/ is automatically created during provisioning)") + assert.Equal(t, 1, numSections, "Expected to have 1 section pre-migration (Main is automatically created during provisioning)") + + if !assert.NoError(t, migrator.MigrateBoards()) { + t.FailNow() + } + + migratedBoards, err := gcsql.GetAllBoards(false) + if !assert.NoError(t, err) { + t.FailNow() + } + migratedSections, err := gcsql.GetAllSections(false) + if !assert.NoError(t, err) { + t.FailNow() + } + + assert.Equal(t, len(migratedBoards), 2, "Expected updated boards list to have two boards") + assert.Equal(t, len(migratedSections), 2, "Expected updated sections list to have two sections") + + // Test migrated sections + mainSection, err := gcsql.GetSectionFromName("Main") + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, "mainmigration", mainSection.Abbreviation, "Expected Main section to have updated abbreviation name 'mainmigration'") + + // Test migrated boards + testBoard, err := gcsql.GetBoardFromDir("test") + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, "Testing Board", testBoard.Title) + assert.Equal(t, "Board for testing pre-2021 migration", testBoard.Subtitle) + testBoardSection, err := gcsql.GetSectionFromID(testBoard.SectionID) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, "Main", testBoardSection.Name, "Expected /test/ board to be in Main section") + + hiddenBoard, err := gcsql.GetBoardFromDir("hidden") + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, "Hidden Board", hiddenBoard.Title) +} + +func TestMigrateBoardsInPlace(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, true) + if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { + t.FailNow() + } + + if !assert.Error(t, migrator.MigrateBoards(), "Not yet implemented") { + t.FailNow() + } +} diff --git a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go index 49bed7a1..045217e9 100644 --- a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go +++ b/cmd/gochan-migration/internal/pre2021/sqlite3_test.go @@ -84,72 +84,3 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 return migrator } - -func TestMigrateBoardsToNewDB(t *testing.T) { - outDir := t.TempDir() - migrator := setupMigrationTest(t, outDir, false) - if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { - t.FailNow() - } - assert.NoError(t, gcsql.ResetBoardSectionArrays()) - - numBoards := len(gcsql.AllBoards) - numSections := len(gcsql.AllSections) - - assert.Equal(t, 1, numBoards, "Expected to have 1 board pre-migration (/test/ is automatically created during provisioning)") - assert.Equal(t, 1, numSections, "Expected to have 1 section pre-migration (Main is automatically created during provisioning)") - - if !assert.NoError(t, migrator.MigrateBoards()) { - t.FailNow() - } - - migratedBoards, err := gcsql.GetAllBoards(false) - if !assert.NoError(t, err) { - t.FailNow() - } - migratedSections, err := gcsql.GetAllSections(false) - if !assert.NoError(t, err) { - t.FailNow() - } - - assert.Equal(t, len(migratedBoards), 2, "Expected updated boards list to have two boards") - assert.Equal(t, len(migratedSections), 2, "Expected updated sections list to have two sections") - - // Test migrated sections - mainSection, err := gcsql.GetSectionFromName("Main") - if !assert.NoError(t, err) { - t.FailNow() - } - assert.Equal(t, "mainmigration", mainSection.Abbreviation, "Expected Main section to have updated abbreviation name 'mainmigration'") - - // Test migrated boards - testBoard, err := gcsql.GetBoardFromDir("test") - if !assert.NoError(t, err) { - t.FailNow() - } - assert.Equal(t, "Testing Board", testBoard.Title) - assert.Equal(t, "Board for testing pre-2021 migration", testBoard.Subtitle) - testBoardSection, err := gcsql.GetSectionFromID(testBoard.SectionID) - if !assert.NoError(t, err) { - t.FailNow() - } - assert.Equal(t, "Main", testBoardSection.Name, "Expected /test/ board to be in Main section") - - hiddenBoard, err := gcsql.GetBoardFromDir("hidden") - if !assert.NoError(t, err) { - t.FailNow() - } - assert.Equal(t, "Hidden Board", hiddenBoard.Title) -} - -func TestMigrateBoardsInPlace(t *testing.T) { - outDir := t.TempDir() - migrator := setupMigrationTest(t, outDir, true) - if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { - t.FailNow() - } - - if !assert.Error(t, migrator.MigrateBoards(), "Not yet implemented") { - t.FailNow() - } -} From 0dcffa06dfc8668604911fb9876d6301107ea7ad Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Thu, 2 Jan 2025 21:31:41 -0800 Subject: [PATCH 039/122] Make createThread function public --- pkg/gcsql/posts.go | 2 +- pkg/gcsql/threads.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/gcsql/posts.go b/pkg/gcsql/posts.go index 486d9031..f52826cb 100644 --- a/pkg/gcsql/posts.go +++ b/pkg/gcsql/posts.go @@ -371,7 +371,7 @@ func (p *Post) Insert(bumpThread bool, boardID int, locked bool, stickied bool, // thread doesn't exist yet, this is a new post p.IsTopPost = true var threadID int - threadID, err = createThread(tx, boardID, locked, stickied, anchored, cyclical) + threadID, err = CreateThread(tx, boardID, locked, stickied, anchored, cyclical) if err != nil { return err } diff --git a/pkg/gcsql/threads.go b/pkg/gcsql/threads.go index 4624da83..b964f7ea 100644 --- a/pkg/gcsql/threads.go +++ b/pkg/gcsql/threads.go @@ -19,7 +19,8 @@ var ( ErrThreadLocked = errors.New("thread is locked and cannot be replied to") ) -func createThread(tx *sql.Tx, boardID int, locked bool, stickied bool, anchored bool, cyclical bool) (threadID int, err error) { +// CreateThread creates a new thread in the database with the given board ID and statuses +func CreateThread(tx *sql.Tx, boardID int, locked bool, stickied bool, anchored bool, cyclical bool) (threadID int, err error) { const lockedQuery = `SELECT locked FROM DBPREFIXboards WHERE id = ?` const insertQuery = `INSERT INTO DBPREFIXthreads (board_id, locked, stickied, anchored, cyclical) VALUES (?,?,?,?,?)` var boardIsLocked bool @@ -32,8 +33,7 @@ func createThread(tx *sql.Tx, boardID int, locked bool, stickied bool, anchored if _, err = ExecTxSQL(tx, insertQuery, boardID, locked, stickied, anchored, cyclical); err != nil { return 0, err } - QueryRowTxSQL(tx, "SELECT MAX(id) FROM DBPREFIXthreads", nil, []any{&threadID}) - return threadID, err + return threadID, QueryRowTxSQL(tx, "SELECT MAX(id) FROM DBPREFIXthreads", nil, []any{&threadID}) } // GetThread returns a a thread object from the database, given its ID From c80970c10e0fd9012f3fc576e9a65a22df876b82 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Thu, 2 Jan 2025 22:28:29 -0800 Subject: [PATCH 040/122] Migrate thread data to new threads table --- .../internal/pre2021/boards.go | 17 +- .../internal/pre2021/posts.go | 220 +++++++----------- .../internal/pre2021/posts_test.go | 36 +++ .../internal/pre2021/pre2021.go | 2 +- .../{sqlite3_test.go => pre2021_test.go} | 5 + .../internal/pre2021/queries.go | 9 +- tools/gochan-pre2021.sqlite3db | Bin 81920 -> 81920 bytes 7 files changed, 141 insertions(+), 148 deletions(-) create mode 100644 cmd/gochan-migration/internal/pre2021/posts_test.go rename cmd/gochan-migration/internal/pre2021/{sqlite3_test.go => pre2021_test.go} (93%) diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index 26b7d7f6..a7b842f3 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -133,6 +133,7 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { for _, board := range allBoards { m.boards = append(m.boards, migrationBoard{ oldSectionID: -1, + oldID: -1, Board: board, }) } @@ -149,7 +150,7 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { var board migrationBoard var maxPages int if err = rows.Scan( - &board.ID, &board.NavbarPosition, &board.Dir, &board.Title, &board.Subtitle, &board.Description, + &board.oldID, &board.NavbarPosition, &board.Dir, &board.Title, &board.Subtitle, &board.Description, &board.SectionID, &board.MaxFilesize, &maxPages, &board.DefaultStyle, &board.Locked, &board.CreatedAt, &board.AnonymousName, &board.ForceAnonymous, &board.AutosageAfter, &board.NoImagesAfter, &board.MaxMessageLength, &board.AllowEmbeds, &board.RedirectToThread, &board.RequireFile, &board.EnableCatalog, @@ -162,9 +163,13 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { for b, newBoard := range m.boards { if newBoard.Dir == board.Dir { - m.boards[b].oldID = board.ID + m.boards[b].oldID = board.oldID m.boards[b].oldSectionID = board.SectionID - common.LogInfo().Str("board", board.Dir).Msg("Board already exists in new db, updating values") + common.LogInfo(). + Str("board", board.Dir). + Int("oldBoardID", board.ID). + Int("migratedBoardID", newBoard.ID). + Msg("Board already exists in new db, updating values") // don't update other values in the array since they don't affect migrating threads or posts if _, err = gcsql.ExecSQL(`UPDATE DBPREFIXboards SET uri = ?, navbar_position = ?, title = ?, subtitle = ?, description = ?, @@ -195,7 +200,11 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { errEv.Err(err).Caller().Str("board", board.Dir).Msg("Failed to create board") return err } - common.LogInfo().Str("board", board.Dir).Msg("Board successfully created") + m.boards = append(m.boards, board) + common.LogInfo(). + Str("dir", board.Dir). + Int("boardID", board.ID). + Msg("Board successfully created") } if err = gcsql.ResetBoardSectionArrays(); err != nil { errEv.Err(err).Caller().Msg("Failed to reset board and section arrays") diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index 751dbf4f..aa004982 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -1,8 +1,6 @@ package pre2021 import ( - "database/sql" - "fmt" "time" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" @@ -10,168 +8,112 @@ import ( ) type postTable struct { - id int - boardid int - parentid int - name string - tripcode string - email string - subject string - message string - message_raw string - password string - filename string - filename_original string - file_checksum string - filesize int - image_w int - image_h int - thumb_w int - thumb_h int - ip string - tag string - timestamp time.Time - autosage bool - deleted_timestamp time.Time - bumped time.Time - stickied bool - locked bool - reviewed bool - - newBoardID int - foundBoard bool - // oldParentID int + gcsql.Post + // id int + // boardID int + // parentID int + // name string + // tripcode string + // email string + // subject string + // message string + // messageRaw string + // password string + filename string + filenameOriginal string + fileChecksum string + filesize int + imageW int + imageH int + thumbW int + thumbH int + // ip string + // tag string + // timestamp time.Time + autosage bool + bumped time.Time + stickied bool + locked bool + // reviewed bool + oldID int + boardID int + oldBoardID int + oldParentID int } func (m *Pre2021Migrator) MigratePosts() error { - var err error - if err = m.migrateThreads(); err != nil { - return err + if m.IsMigratingInPlace() { + return m.migratePostsInPlace() } - return m.migratePostsUtil() + return m.migratePostsToNewDB() } -func (m *Pre2021Migrator) migrateThreads() error { - tx, err := m.db.Begin() - if err != nil { - return err - } +func (m *Pre2021Migrator) migratePostsToNewDB() error { + errEv := common.LogError() + defer errEv.Discard() - stmt, err := m.db.PrepareSQL(postsQuery, tx) + tx, err := gcsql.BeginTx() if err != nil { - tx.Rollback() + errEv.Err(err).Caller().Msg("Failed to start transaction") return err } - rows, err := stmt.Query() - if err != nil && err != sql.ErrNoRows { - tx.Rollback() + defer tx.Rollback() + + rows, err := m.db.QuerySQL(threadsQuery) + if err != nil { + errEv.Err(err).Caller().Msg("Failed to get threads") return err } - defer stmt.Close() defer rows.Close() + + var threadIDsWithInvalidBoards []int + var missingBoardIDs []int for rows.Next() { - var post postTable + var thread postTable if err = rows.Scan( - &post.id, - &post.boardid, - &post.parentid, - &post.name, - &post.tripcode, - &post.email, - &post.subject, - &post.message, - &post.message_raw, - &post.password, - &post.filename, - &post.filename_original, - &post.file_checksum, - &post.filesize, - &post.image_w, - &post.image_h, - &post.thumb_w, - &post.thumb_h, - &post.ip, - &post.tag, - &post.timestamp, - &post.autosage, - &post.deleted_timestamp, - &post.bumped, - &post.stickied, - &post.locked, - &post.reviewed, + &thread.oldID, &thread.oldBoardID, &thread.oldParentID, &thread.Name, &thread.Tripcode, &thread.Email, + &thread.Subject, &thread.Message, &thread.MessageRaw, &thread.Password, &thread.filename, + &thread.filenameOriginal, &thread.fileChecksum, &thread.filesize, &thread.imageW, &thread.imageH, + &thread.thumbW, &thread.thumbH, &thread.IP, &thread.CreatedOn, &thread.autosage, + &thread.bumped, &thread.stickied, &thread.locked, ); err != nil { - tx.Rollback() + errEv.Err(err).Caller().Msg("Failed to scan thread") return err } - var postBoardDir string - for _, newBoard := range gcsql.AllBoards { - if newBoard.Dir == postBoardDir { - post.newBoardID = newBoard.ID - post.foundBoard = true + var foundBoard bool + for _, board := range m.boards { + if board.oldID == thread.oldBoardID { + thread.boardID = board.ID + foundBoard = true + break } } - if !post.foundBoard { - common.LogWarning().Int("boardID", post.boardid). - Msg("Pre-migrated post has an invalid boardid (board doesn't exist), skipping") + if !foundBoard { + threadIDsWithInvalidBoards = append(threadIDsWithInvalidBoards, thread.oldID) + missingBoardIDs = append(missingBoardIDs, thread.oldBoardID) continue } - // var stmt *sql.Stmt - // var err error - preparedStr, _ := gcsql.SetupSQLString(`SELECT id FROM DBPREFIXboards WHERE ui = ?`, m.db) - stmt, err := tx.Prepare(preparedStr) - if err != nil { - tx.Rollback() - return err + if thread.ThreadID, err = gcsql.CreateThread(tx, thread.boardID, thread.locked, thread.stickied, thread.autosage, false); err != nil { + errEv.Err(err).Caller(). + Int("boardID", thread.boardID). + Msg("Failed to create thread") } - stmt.QueryRow(post.boardid).Scan(&post.newBoardID) - - // gcsql.QueryRowSQL(`SELECT id FROM DBPREFIXboards WHERE uri = ?`, []interface{}{}) - if post.parentid == 0 { - // post is a thread, save it to the DBPREFIXthreads table - // []interfaceP{{post.newParentID} - - if err = gcsql.QueryRowSQL( - `SELECT board_id FROM DBPREFIXthreads ORDER BY board_id LIMIT 1`, - nil, - []interface{}{&post.newBoardID}, - ); err != nil { - tx.Rollback() - return err - } - fmt.Println("Current board ID:", post.newBoardID) - prepareStr, _ := gcsql.SetupSQLString( - `INSERT INTO DBPREFIXthreads - (board_id, locked, stickied) - VALUES(?, ?, ?)`, m.db) - stmt, err = tx.Prepare(prepareStr) - if err != nil { - tx.Rollback() - return err - } - stmt.Exec(post.newBoardID, post.locked, post.stickied) - // // stmt, err := db.Prepare("INSERT table SET unique_id=? ON DUPLICATE KEY UPDATE id=LAST_INSERT_ID(id)") - // gcsql.ExecSQL(`INSERT INTO DBPREFIXthreads (board_id) VALUES(?)`, post.newBoardID) - - // /* - // id - // board_id - // locked - // stickied - // anchored - // cyclical - // last_bump - // deleted_at - // is_deleted - - // */ - - } - m.posts = append(m.posts, post) } - return tx.Commit() + if len(threadIDsWithInvalidBoards) > 0 { + errEv.Caller(). + Ints("threadIDs", threadIDsWithInvalidBoards). + Ints("boardIDs", missingBoardIDs). + Msg("Failed to find boards for threads") + return common.NewMigrationError("pre2021", "Found threads with missing boards") + } + + if err = tx.Commit(); err != nil { + errEv.Err(err).Caller().Msg("Failed to commit transaction") + } + return err } -func (*Pre2021Migrator) migratePostsUtil() error { - return nil +func (m *Pre2021Migrator) migratePostsInPlace() error { + return common.NewMigrationError("pre2021", "not yet implemented") } diff --git a/cmd/gochan-migration/internal/pre2021/posts_test.go b/cmd/gochan-migration/internal/pre2021/posts_test.go new file mode 100644 index 00000000..a04ab5f6 --- /dev/null +++ b/cmd/gochan-migration/internal/pre2021/posts_test.go @@ -0,0 +1,36 @@ +package pre2021 + +import ( + "testing" + + "github.com/gochan-org/gochan/pkg/gcsql" + "github.com/stretchr/testify/assert" +) + +func TestMigratePostsToNewDB(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, false) + if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateBoards()) { + t.FailNow() + } + + var numThreads int + if !assert.NoError(t, migrator.db.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXposts WHERE parentid = 0 AND deleted_timestamp IS NULL", nil, []any{&numThreads}), "Failed to get number of threads") { + t.FailNow() + } + assert.Equal(t, 2, numThreads, "Expected to have two threads pre-migration") + + if !assert.NoError(t, migrator.MigratePosts()) { + t.FailNow() + } + + var numMigratedThreads int + if !assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numMigratedThreads}), "Failed to get number of migrated threads") { + t.FailNow() + } + assert.Equal(t, 2, numMigratedThreads, "Expected to have three migrated threads") +} diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index 8892d75e..9b19a3c3 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -77,7 +77,7 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) { errEv.Caller().Err(err).Msg("Failed to migrate boards") return false, err } - common.LogInfo().Msg("Migrated boards") + common.LogInfo().Msg("Migrated boards successfully") // if err = m.MigratePosts(); err != nil { // return false, err // } diff --git a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go b/cmd/gochan-migration/internal/pre2021/pre2021_test.go similarity index 93% rename from cmd/gochan-migration/internal/pre2021/sqlite3_test.go rename to cmd/gochan-migration/internal/pre2021/pre2021_test.go index 045217e9..e28eb26c 100644 --- a/cmd/gochan-migration/internal/pre2021/sqlite3_test.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021_test.go @@ -6,6 +6,7 @@ import ( "path" "testing" + "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/config" "github.com/gochan-org/gochan/pkg/gcsql" "github.com/gochan-org/gochan/pkg/gcutil/testutil" @@ -21,6 +22,10 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 if !assert.NoError(t, err) { t.FailNow() } + if !assert.NoError(t, common.InitTestMigrationLog(t)) { + t.FailNow() + } + dbName := "gochan-pre2021.sqlite3db" dbHost := path.Join(dir, sqlite3DBDir, dbName) migratedDBName := "gochan-migrated.sqlite3db" diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index fe0ae120..f486de93 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -1,7 +1,7 @@ package pre2021 const ( - sectionsQuery = `SELECT id, list_order, hidden, name, abbreviation FROM DBPREFIXsections` + sectionsQuery = "SELECT id, list_order, hidden, name, abbreviation FROM DBPREFIXsections" boardsQuery = `SELECT id, list_order, dir, title, subtitle, description, section, max_file_size, max_pages, default_style, locked, created_on, anonymous, forced_anon, autosage_after, no_images_after, max_message_length, embeds_allowed, @@ -9,7 +9,8 @@ redirect_to_thread, require_file, enable_catalog FROM DBPREFIXboards` postsQuery = `SELECT id, boardid, parentid, name, tripcode, email, subject, message, message_raw, password, filename, -filename_original, file_checksum, filesize, image_w, image_h, thumb_w, thumb_h, ip, tag, timestamp, autosage, deleted_timestamp, -bumped, stickied, locked, reviewed -FROM DBPREFIXposts WHERE deleted_timestamp = NULL` +filename_original, file_checksum, filesize, image_w, image_h, thumb_w, thumb_h, ip, timestamp, autosage, +bumped, stickied, locked FROM DBPREFIXposts WHERE deleted_timestamp IS NULL` + + threadsQuery = postsQuery + " AND parentid = 0" ) diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db index 632d3be5712b5d68cd31bb3a3f185a2a330716cc..e4a52098c3b648d5ae18a54bbe9f114fcaa596c0 100644 GIT binary patch delta 626 zcmZo@U~On%ogmF9H&Mo!QEp>GoIWGl=KcCk4Z^(N7}$6U82A-<=kjgh6XyBFQ?Riy zhNr%folS;;gOL{q0s}!L2MF-;8gnoh0~xF#8JWcjK$NIZkY8M)keLT$rKIMhmZYXA zlw=g8CZ-TpVPs%rs%v1VYhbKkU}9xzZe?Hy6RJlNYgAyBVPFK>&8ul^3wMet&?&|s zr!cwTuojndP{j=`jr0u7Ec8sx^x(D|7$<>UWNKw>0&@}6L1tEl=4e77RpCrB44j-G z#|aAy14&R|ae$l%4z2RU5+EulPeHb9@~?erEDQ_`vJ)L8>l Date: Thu, 2 Jan 2025 23:15:23 -0800 Subject: [PATCH 041/122] Add thread post migration for pre2021 --- .../internal/pre2021/posts.go | 77 +++++++++++++----- .../internal/pre2021/posts_test.go | 6 ++ pkg/gcsql/posts.go | 29 ++++--- tools/gochan-pre2021.sqlite3db | Bin 81920 -> 81920 bytes 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index aa004982..59bff89c 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -1,6 +1,7 @@ package pre2021 import ( + "context" "time" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" @@ -9,16 +10,11 @@ import ( type postTable struct { gcsql.Post - // id int - // boardID int - // parentID int - // name string - // tripcode string - // email string - // subject string - // message string - // messageRaw string - // password string + autosage bool + bumped time.Time + stickied bool + locked bool + filename string filenameOriginal string fileChecksum string @@ -27,14 +23,7 @@ type postTable struct { imageH int thumbW int thumbH int - // ip string - // tag string - // timestamp time.Time - autosage bool - bumped time.Time - stickied bool - locked bool - // reviewed bool + oldID int boardID int oldBoardID int @@ -94,11 +83,61 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { continue } - if thread.ThreadID, err = gcsql.CreateThread(tx, thread.boardID, thread.locked, thread.stickied, thread.autosage, false); err != nil { + // create the thread as not locked so migration replies can be inserted. It will be locked after they are all inserted + if thread.ThreadID, err = gcsql.CreateThread(tx, thread.boardID, false, thread.stickied, thread.autosage, false); err != nil { errEv.Err(err).Caller(). Int("boardID", thread.boardID). Msg("Failed to create thread") } + + // insert thread top post + if err = thread.InsertWithContext(context.Background(), tx, true, thread.boardID, false, thread.stickied, thread.autosage, false); err != nil { + errEv.Err(err).Caller(). + Int("boardID", thread.boardID). + Int("threadID", thread.ThreadID). + Msg("Failed to insert thread OP") + } + + // get and insert replies + replyRows, err := m.db.QuerySQL(postsQuery+" AND parentid = ?", thread.oldID) + if err != nil { + errEv.Err(err).Caller(). + Int("parentID", thread.oldID). + Msg("Failed to get reply rows") + return err + } + defer replyRows.Close() + + for replyRows.Next() { + var reply postTable + if err = replyRows.Scan( + &reply.oldID, &reply.oldBoardID, &reply.oldParentID, &reply.Name, &reply.Tripcode, &reply.Email, + &reply.Subject, &reply.Message, &reply.MessageRaw, &reply.Password, &reply.filename, + &reply.filenameOriginal, &reply.fileChecksum, &reply.filesize, &reply.imageW, &reply.imageH, + &reply.thumbW, &reply.thumbH, &reply.IP, &reply.CreatedOn, &reply.autosage, + &reply.bumped, &reply.stickied, &reply.locked, + ); err != nil { + errEv.Err(err).Caller(). + Int("parentID", thread.oldID). + Msg("Failed to scan reply") + return err + } + reply.ThreadID = thread.ThreadID + if err = reply.InsertWithContext(context.Background(), tx, true, reply.boardID, false, false, false, false); err != nil { + errEv.Err(err).Caller(). + Int("parentID", thread.oldID). + Msg("Failed to insert reply post") + return err + } + } + + if thread.locked { + if _, err = gcsql.ExecTxSQL(tx, "UPDATE DBPREFIXthreads SET locked = TRUE WHERE id = ?", thread.ThreadID); err != nil { + errEv.Err(err).Caller(). + Int("threadID", thread.ThreadID). + Msg("Unable to re-lock migrated thread") + } + } } if len(threadIDsWithInvalidBoards) > 0 { errEv.Caller(). diff --git a/cmd/gochan-migration/internal/pre2021/posts_test.go b/cmd/gochan-migration/internal/pre2021/posts_test.go index a04ab5f6..fa0cced6 100644 --- a/cmd/gochan-migration/internal/pre2021/posts_test.go +++ b/cmd/gochan-migration/internal/pre2021/posts_test.go @@ -33,4 +33,10 @@ func TestMigratePostsToNewDB(t *testing.T) { t.FailNow() } assert.Equal(t, 2, numMigratedThreads, "Expected to have three migrated threads") + + var locked bool + if !assert.NoError(t, gcsql.QueryRowSQL("SELECT locked FROM DBPREFIXthreads WHERE id = 1", nil, []any{&locked})) { + t.FailNow() + } + assert.True(t, locked, "Expected thread ID 1 to be locked") } diff --git a/pkg/gcsql/posts.go b/pkg/gcsql/posts.go index f52826cb..3fa1bc5d 100644 --- a/pkg/gcsql/posts.go +++ b/pkg/gcsql/posts.go @@ -347,7 +347,7 @@ func (p *Post) Delete() error { return err } -func (p *Post) Insert(bumpThread bool, boardID int, locked bool, stickied bool, anchored bool, cyclical bool) error { +func (p *Post) InsertWithContext(ctx context.Context, tx *sql.Tx, bumpThread bool, boardID int, locked bool, stickied bool, anchored bool, cyclical bool) error { if p.ID > 0 { // already inserted return ErrorPostAlreadySent @@ -358,15 +358,7 @@ func (p *Post) Insert(bumpThread bool, boardID int, locked bool, stickied bool, VALUES(?,?,PARAM_ATON,CURRENT_TIMESTAMP,?,?,?,?,?,?,?,?,?,?)` bumpSQL := `UPDATE DBPREFIXthreads SET last_bump = CURRENT_TIMESTAMP WHERE id = ?` - ctx, cancel := context.WithTimeout(context.Background(), gcdb.defaultTimeout) - defer cancel() - - tx, err := BeginContextTx(ctx) - if err != nil { - return err - } - defer tx.Rollback() - + var err error if p.ThreadID == 0 { // thread doesn't exist yet, this is a new post p.IsTopPost = true @@ -401,6 +393,23 @@ func (p *Post) Insert(bumpThread bool, boardID int, locked bool, stickied bool, return err } } + return nil +} + +func (p *Post) Insert(bumpThread bool, boardID int, locked bool, stickied bool, anchored bool, cyclical bool) error { + ctx, cancel := context.WithTimeout(context.Background(), gcdb.defaultTimeout) + defer cancel() + + tx, err := BeginContextTx(ctx) + if err != nil { + return err + } + defer tx.Rollback() + + if err = p.InsertWithContext(ctx, tx, bumpThread, boardID, locked, stickied, anchored, cyclical); err != nil { + return err + } + return tx.Commit() } diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db index e4a52098c3b648d5ae18a54bbe9f114fcaa596c0..ff34a77fb6a26e974421ac3c258a2fa740fa4941 100644 GIT binary patch delta 90 zcmZo@U~On%ogmF9KT*b+QGR2>qV-&J8Q6FV82A-8jWNe&QDlxF8kq delta 88 zcmZo@U~On%ogmF9H&Mo!QEp?xqV-(g7}$6U82A-<=WgcNFp*a|oJod(lam(+goTBH mBnJqn@)~n67y}vQi6!NUlke^|V_{%mke%o#xmkGM+eQGY{1s6E From 7cddc3e43e5e406bd683c8119829ceb2ea9462ac Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Fri, 3 Jan 2025 21:14:16 -0800 Subject: [PATCH 042/122] Make pre-migration /test/ have board id 3, make sure no deleted threads were migrated --- .../internal/pre2021/boards_test.go | 4 +++- .../internal/pre2021/posts_test.go | 5 +++++ tools/gochan-pre2021.sqlite3db | Bin 81920 -> 81920 bytes 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/gochan-migration/internal/pre2021/boards_test.go b/cmd/gochan-migration/internal/pre2021/boards_test.go index f65b37d7..2f06e282 100644 --- a/cmd/gochan-migration/internal/pre2021/boards_test.go +++ b/cmd/gochan-migration/internal/pre2021/boards_test.go @@ -34,7 +34,7 @@ func TestMigrateBoardsToNewDB(t *testing.T) { t.FailNow() } - assert.Equal(t, len(migratedBoards), 2, "Expected updated boards list to have two boards") + assert.Equal(t, len(migratedBoards), 3, "Expected updated boards list to have three boards") assert.Equal(t, len(migratedSections), 2, "Expected updated sections list to have two sections") // Test migrated sections @@ -49,8 +49,10 @@ func TestMigrateBoardsToNewDB(t *testing.T) { if !assert.NoError(t, err) { t.FailNow() } + assert.Equal(t, 1, testBoard.ID) assert.Equal(t, "Testing Board", testBoard.Title) assert.Equal(t, "Board for testing pre-2021 migration", testBoard.Subtitle) + assert.Equal(t, "Board for testing pre-2021 migration description", testBoard.Description) testBoardSection, err := gcsql.GetSectionFromID(testBoard.SectionID) if !assert.NoError(t, err) { t.FailNow() diff --git a/cmd/gochan-migration/internal/pre2021/posts_test.go b/cmd/gochan-migration/internal/pre2021/posts_test.go index fa0cced6..93e16b69 100644 --- a/cmd/gochan-migration/internal/pre2021/posts_test.go +++ b/cmd/gochan-migration/internal/pre2021/posts_test.go @@ -39,4 +39,9 @@ func TestMigratePostsToNewDB(t *testing.T) { t.FailNow() } assert.True(t, locked, "Expected thread ID 1 to be locked") + + // make sure deleted posts and threads weren't migrated + var numDeleted int + assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXposts WHERE message_raw LIKE '%deleted%' OR is_deleted", nil, []any{&numDeleted})) + assert.Zero(t, numDeleted, "Expected no deleted threads to be migrated") } diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db index ff34a77fb6a26e974421ac3c258a2fa740fa4941..a78f7c80315e33f8a710fb5800aa3aafc949b278 100644 GIT binary patch delta 425 zcmZo@U~On%ogmGqJW<6;@P znE9VD@c-a{%l~AvpujbLX%1#l#*)5JYl-057jG2ZJ$?!8EylPxj4vH&%30#&M7S_;E7r~)+@ UgETO^O!nNTI5}ir+@b>w0EMVyVE_OC delta 262 zcmZo@U~On%ogmF9KT*b+QGR1Wu0A8v=5zXH3LH#)Y7Bg8e783?O7RId8Zk;TaENkn zsE6iqGBe6^80${H7F)D=bzCGP2NVBW2L8AFKQ;>rJmeSRVwPmg$V^E|&0}I`7Ue8S zE#CY;pUFX(cP;}PPXPnJ0`CsKO?<*Uzjz8ZHpcLghiv=jzX93Y^| zYs_J63}j4h*xxXD)4n|U##lz6Iu21$eM5VDVOeuoCPpAo6*cB$Fy`dsn7n@Xq0Nj3 H{?`Kldu&8u From ee0b91059fdf6f5b4b835f734629d8c654ac8200 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Fri, 3 Jan 2025 22:01:16 -0800 Subject: [PATCH 043/122] Add pre-2021 post upload migration --- .../internal/pre2021/posts.go | 72 ++++++++++++++----- .../internal/pre2021/posts_test.go | 4 ++ .../internal/pre2021/pre2021.go | 19 +++-- pkg/gcsql/uploads.go | 36 ++++------ 4 files changed, 83 insertions(+), 48 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index 59bff89c..44389776 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -2,10 +2,13 @@ package pre2021 import ( "context" + "database/sql" "time" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/gcsql" + "github.com/gochan-org/gochan/pkg/gcutil" + "github.com/rs/zerolog" ) type postTable struct { @@ -37,6 +40,47 @@ func (m *Pre2021Migrator) MigratePosts() error { return m.migratePostsToNewDB() } +func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *postTable, errEv *zerolog.Event) error { + var err error + + if post.oldParentID == 0 { + // migrating post was a thread OP, create the row in the threads table + if post.ThreadID, err = gcsql.CreateThread(tx, post.boardID, false, post.stickied, post.autosage, false); err != nil { + errEv.Err(err).Caller(). + Int("boardID", post.boardID). + Msg("Failed to create thread") + } + } + + // insert thread top post + if err = post.InsertWithContext(context.Background(), tx, true, post.boardID, false, post.stickied, post.autosage, false); err != nil { + errEv.Err(err).Caller(). + Int("boardID", post.boardID). + Int("threadID", post.ThreadID). + Msg("Failed to insert thread OP") + } + + if post.filename != "" { + if err = post.AttachFileTx(tx, &gcsql.Upload{ + PostID: post.ID, + OriginalFilename: post.filenameOriginal, + Filename: post.filename, + Checksum: post.fileChecksum, + FileSize: post.filesize, + ThumbnailWidth: post.thumbW, + ThumbnailHeight: post.thumbH, + Width: post.imageW, + Height: post.imageH, + }); err != nil { + errEv.Err(err).Caller(). + Int("oldPostID", post.oldID). + Msg("Failed to attach upload to migrated post") + return err + } + } + return nil +} + func (m *Pre2021Migrator) migratePostsToNewDB() error { errEv := common.LogError() defer errEv.Discard() @@ -57,6 +101,7 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { var threadIDsWithInvalidBoards []int var missingBoardIDs []int + var migratedThreads int for rows.Next() { var thread postTable if err = rows.Scan( @@ -83,19 +128,8 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { continue } - // create the thread as not locked so migration replies can be inserted. It will be locked after they are all inserted - if thread.ThreadID, err = gcsql.CreateThread(tx, thread.boardID, false, thread.stickied, thread.autosage, false); err != nil { - errEv.Err(err).Caller(). - Int("boardID", thread.boardID). - Msg("Failed to create thread") - } - - // insert thread top post - if err = thread.InsertWithContext(context.Background(), tx, true, thread.boardID, false, thread.stickied, thread.autosage, false); err != nil { - errEv.Err(err).Caller(). - Int("boardID", thread.boardID). - Int("threadID", thread.ThreadID). - Msg("Failed to insert thread OP") + if err = m.migratePost(tx, &thread, errEv); err != nil { + return err } // get and insert replies @@ -123,10 +157,7 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { return err } reply.ThreadID = thread.ThreadID - if err = reply.InsertWithContext(context.Background(), tx, true, reply.boardID, false, false, false, false); err != nil { - errEv.Err(err).Caller(). - Int("parentID", thread.oldID). - Msg("Failed to insert reply post") + if err = m.migratePost(tx, &reply, errEv); err != nil { return err } } @@ -138,6 +169,7 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { Msg("Unable to re-lock migrated thread") } } + migratedThreads++ } if len(threadIDsWithInvalidBoards) > 0 { errEv.Caller(). @@ -149,8 +181,12 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { if err = tx.Commit(); err != nil { errEv.Err(err).Caller().Msg("Failed to commit transaction") + return err } - return err + gcutil.LogInfo(). + Int("migratedThreads", migratedThreads). + Msg("Migrated threads successfully") + return nil } func (m *Pre2021Migrator) migratePostsInPlace() error { diff --git a/cmd/gochan-migration/internal/pre2021/posts_test.go b/cmd/gochan-migration/internal/pre2021/posts_test.go index 93e16b69..d55d8d4e 100644 --- a/cmd/gochan-migration/internal/pre2021/posts_test.go +++ b/cmd/gochan-migration/internal/pre2021/posts_test.go @@ -44,4 +44,8 @@ func TestMigratePostsToNewDB(t *testing.T) { var numDeleted int assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXposts WHERE message_raw LIKE '%deleted%' OR is_deleted", nil, []any{&numDeleted})) assert.Zero(t, numDeleted, "Expected no deleted threads to be migrated") + + var numUploadPosts int + assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXfiles", nil, []any{&numUploadPosts})) + assert.Equal(t, 1, numUploadPosts, "Expected to have 1 upload post") } diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index 9b19a3c3..e81b6d0c 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -22,14 +22,14 @@ type Pre2021Migrator struct { options *common.MigrationOptions config Pre2021Config - posts []postTable boards []migrationBoard sections []migrationSection } // IsMigratingInPlace implements common.DBMigrator. func (m *Pre2021Migrator) IsMigratingInPlace() bool { - return m.config.DBname == config.GetSQLConfig().DBname + sqlConfig := config.GetSQLConfig() + return m.config.DBname == sqlConfig.DBname && m.config.DBhost == sqlConfig.DBhost && m.config.DBprefix == sqlConfig.DBprefix } func (m *Pre2021Migrator) readConfig() error { @@ -56,7 +56,13 @@ func (m *Pre2021Migrator) Init(options *common.MigrationOptions) error { func (m *Pre2021Migrator) IsMigrated() (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(m.config.DBTimeoutSeconds)*time.Second) defer cancel() - return common.TableExists(ctx, m.db, nil, "DBPREFIXdatabase_version", &m.config.SQLConfig) + var sqlConfig config.SQLConfig + if m.IsMigratingInPlace() { + sqlConfig = config.GetSQLConfig() + } else { + sqlConfig = m.config.SQLConfig + } + return common.TableExists(ctx, m.db, nil, "DBPREFIXdatabase_version", &sqlConfig) } func (m *Pre2021Migrator) MigrateDB() (bool, error) { @@ -74,13 +80,12 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) { } if err := m.MigrateBoards(); err != nil { - errEv.Caller().Err(err).Msg("Failed to migrate boards") return false, err } common.LogInfo().Msg("Migrated boards successfully") - // if err = m.MigratePosts(); err != nil { - // return false, err - // } + if err = m.MigratePosts(); err != nil { + return false, err + } // if err = m.MigrateStaff("password"); err != nil { // return false, err // } diff --git a/pkg/gcsql/uploads.go b/pkg/gcsql/uploads.go index 7b1eb1d7..aee41172 100644 --- a/pkg/gcsql/uploads.go +++ b/pkg/gcsql/uploads.go @@ -49,7 +49,7 @@ func (p *Post) nextFileOrder() (int, error) { return next, err } -func (p *Post) AttachFile(upload *Upload) error { +func (p *Post) AttachFileTx(tx *sql.Tx, upload *Upload) error { if upload == nil { return nil // no upload to attach, so no error } @@ -69,38 +69,28 @@ func (p *Post) AttachFile(upload *Upload) error { if upload.ID > 0 { return ErrAlreadyAttached } - tx, err := BeginTx() - if err != nil { - return err - } - defer tx.Rollback() - stmt, err := PrepareSQL(insertSQL, tx) - if err != nil { - return err - } - defer stmt.Close() - - if upload.FileOrder < 1 { - upload.FileOrder, err = p.nextFileOrder() - if err != nil { - return err - } - } - upload.PostID = p.ID - if _, err = stmt.Exec( + if _, err = ExecTxSQL(tx, insertSQL, &upload.PostID, &upload.FileOrder, &upload.OriginalFilename, &upload.Filename, &upload.Checksum, &upload.FileSize, &upload.IsSpoilered, &upload.ThumbnailWidth, &upload.ThumbnailHeight, &upload.Width, &upload.Height, ); err != nil { return err } - if upload.ID, err = getLatestID("DBPREFIXfiles", tx); err != nil { + + upload.ID, err = getLatestID("DBPREFIXfiles", tx) + return err +} + +func (p *Post) AttachFile(upload *Upload) error { + tx, err := BeginTx() + if err != nil { return err } - if err = tx.Commit(); err != nil { + defer tx.Rollback() + if err = p.AttachFileTx(tx, upload); err != nil { return err } - return stmt.Close() + return tx.Commit() } // GetUploadFilenameAndBoard returns the filename (or an empty string) and From 9ca424f4da9b243a082185951d7390a2993f2347 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Fri, 3 Jan 2025 23:21:16 -0800 Subject: [PATCH 044/122] Ad test for pre2021 ban migration and start adding implementation --- cmd/gochan-migration/internal/pre2021/bans.go | 124 ++++++++++++++++++ .../internal/pre2021/bans_test.go | 27 ++++ .../internal/pre2021/posts.go | 8 +- .../internal/pre2021/pre2021.go | 10 +- .../internal/pre2021/queries.go | 3 + pkg/gcsql/bans.go | 31 ++--- 6 files changed, 177 insertions(+), 26 deletions(-) create mode 100644 cmd/gochan-migration/internal/pre2021/bans.go create mode 100644 cmd/gochan-migration/internal/pre2021/bans_test.go diff --git a/cmd/gochan-migration/internal/pre2021/bans.go b/cmd/gochan-migration/internal/pre2021/bans.go new file mode 100644 index 00000000..a7dafb47 --- /dev/null +++ b/cmd/gochan-migration/internal/pre2021/bans.go @@ -0,0 +1,124 @@ +package pre2021 + +import ( + "errors" + "net" + "strings" + "time" + + "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" + "github.com/gochan-org/gochan/pkg/gcsql" + "github.com/gochan-org/gochan/pkg/gcutil" +) + +type migrationBan struct { + oldID int + allowRead string + ip string + name string + nameIsRegex bool + filename string + fileChecksum string + boards string + staff string + timestamp time.Time + expires time.Time + permaban bool + reason string + banType int + staffNote string + appealAt time.Time + canAppeal bool + + boardIDs []int + banID int + staffID int + filterID int +} + +func (m *Pre2021Migrator) migrateBansInPlace() error { + return common.NewMigrationError("pre2021", "migrateBansInPlace not yet implemented") +} + +func (m *Pre2021Migrator) migrateBansToNewDB() error { + errEv := common.LogError() + defer errEv.Discard() + + tx, err := gcsql.BeginTx() + if err != nil { + errEv.Err(err).Caller().Msg("Failed to start transaction") + return err + } + defer tx.Rollback() + + rows, err := m.db.QuerySQL(bansQuery) + if err != nil { + errEv.Err(err).Caller().Msg("Failed to get bans") + return err + } + defer rows.Close() + + for rows.Next() { + var ban migrationBan + if err = rows.Scan( + &ban.oldID, &ban.allowRead, &ban.ip, &ban.name, &ban.nameIsRegex, &ban.filename, &ban.fileChecksum, + &ban.boards, &ban.staff, &ban.timestamp, &ban.expires, &ban.permaban, &ban.reason, &ban.banType, &ban.staffNote, &ban.appealAt, &ban.canAppeal, + ); err != nil { + errEv.Err(err).Caller().Msg("Failed to scan ban row") + return err + } + + if ban.boards != "" && ban.boards != "*" { + boardDirs := strings.Split(ban.boards, ",") + for _, dir := range boardDirs { + dir = strings.TrimSpace(dir) + boardID, err := gcsql.GetBoardIDFromDir(dir) + if err != nil { + if errors.Is(err, gcsql.ErrBoardDoesNotExist) { + common.Logger().Warn().Str("board", dir).Msg("Found unrecognized ban board") + continue + } else { + errEv.Err(err).Caller().Str("board", dir).Msg("Failed getting board ID from dir") + return err + } + } + ban.boardIDs = append(ban.boardIDs, boardID) + } + } + + if len(ban.boardIDs) == 0 { + if ban.ip != "" { + if net.ParseIP(ban.ip) == nil { + gcutil.LogWarning(). + Int("oldID", ban.oldID). + Str("ip", ban.ip). + Msg("Found ban with invalid IP address, skipping") + continue + } + migratedBan := &gcsql.IPBan{ + BoardID: nil, + RangeStart: ban.ip, + RangeEnd: ban.ip, + IssuedAt: ban.timestamp, + } + migratedBan.CanAppeal = ban.canAppeal + migratedBan.AppealAt = ban.appealAt + migratedBan.ExpiresAt = ban.expires + migratedBan.Permanent = ban.permaban + migratedBan.Message = ban.reason + migratedBan.StaffNote = ban.staffNote + gcsql.NewIPBanTx(tx, migratedBan) + } + } + + } + + return nil +} + +func (m *Pre2021Migrator) MigrateBans() error { + if m.IsMigratingInPlace() { + return m.migrateBansInPlace() + } + return m.migrateBansToNewDB() +} diff --git a/cmd/gochan-migration/internal/pre2021/bans_test.go b/cmd/gochan-migration/internal/pre2021/bans_test.go new file mode 100644 index 00000000..1662e354 --- /dev/null +++ b/cmd/gochan-migration/internal/pre2021/bans_test.go @@ -0,0 +1,27 @@ +package pre2021 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMigrateBansToNewDB(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, false) + if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateBoards()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigratePosts()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateBans()) { + t.FailNow() + } +} diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index 44389776..b10e5bd0 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -11,7 +11,7 @@ import ( "github.com/rs/zerolog" ) -type postTable struct { +type migrationPost struct { gcsql.Post autosage bool bumped time.Time @@ -40,7 +40,7 @@ func (m *Pre2021Migrator) MigratePosts() error { return m.migratePostsToNewDB() } -func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *postTable, errEv *zerolog.Event) error { +func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *zerolog.Event) error { var err error if post.oldParentID == 0 { @@ -103,7 +103,7 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { var missingBoardIDs []int var migratedThreads int for rows.Next() { - var thread postTable + var thread migrationPost if err = rows.Scan( &thread.oldID, &thread.oldBoardID, &thread.oldParentID, &thread.Name, &thread.Tripcode, &thread.Email, &thread.Subject, &thread.Message, &thread.MessageRaw, &thread.Password, &thread.filename, @@ -143,7 +143,7 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { defer replyRows.Close() for replyRows.Next() { - var reply postTable + var reply migrationPost if err = replyRows.Scan( &reply.oldID, &reply.oldBoardID, &reply.oldParentID, &reply.Name, &reply.Tripcode, &reply.Email, &reply.Subject, &reply.Message, &reply.MessageRaw, &reply.Password, &reply.filename, diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index e81b6d0c..7d80e3a3 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -89,9 +89,9 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) { // if err = m.MigrateStaff("password"); err != nil { // return false, err // } - // if err = m.MigrateBans(); err != nil { - // return false, err - // } + if err = m.MigrateBans(); err != nil { + return false, err + } // if err = m.MigrateAnnouncements(); err != nil { // return false, err // } @@ -103,10 +103,6 @@ func (*Pre2021Migrator) MigrateStaff(_ string) error { return nil } -func (*Pre2021Migrator) MigrateBans() error { - return nil -} - func (*Pre2021Migrator) MigrateAnnouncements() error { return nil } diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index f486de93..12069581 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -13,4 +13,7 @@ filename_original, file_checksum, filesize, image_w, image_h, thumb_w, thumb_h, bumped, stickied, locked FROM DBPREFIXposts WHERE deleted_timestamp IS NULL` threadsQuery = postsQuery + " AND parentid = 0" + + bansQuery = `SELECT id, allow_read, COALESCE(ip, '') as ip, name, name_is_regex, filename, file_checksum, boards, staff, +timestamp, expires, permaban, reason, type, staff_note, appeal_at, can_appeal FROM DBPREFIXbanlist` ) diff --git a/pkg/gcsql/bans.go b/pkg/gcsql/bans.go index 0a122b02..2632fcd3 100644 --- a/pkg/gcsql/bans.go +++ b/pkg/gcsql/bans.go @@ -25,7 +25,7 @@ type Ban interface { Deactivate(int) error } -func NewIPBan(ban *IPBan) error { +func NewIPBanTx(tx *sql.Tx, ban *IPBan) error { const query = `INSERT INTO DBPREFIXip_ban (staff_id, board_id, banned_for_post_id, copy_post_text, is_thread_ban, is_active, range_start, range_end, appeal_at, expires_at, @@ -34,27 +34,28 @@ func NewIPBan(ban *IPBan) error { if ban.ID > 0 { return ErrBanAlreadyInserted } + _, err := ExecTxSQL(tx, query, ban.StaffID, ban.BoardID, ban.BannedForPostID, ban.CopyPostText, + ban.IsThreadBan, ban.IsActive, ban.RangeStart, ban.RangeEnd, ban.AppealAt, + ban.ExpiresAt, ban.Permanent, ban.StaffNote, ban.Message, ban.CanAppeal) + if err != nil { + return err + } + + ban.ID, err = getLatestID("DBPREFIXip_ban", tx) + return err +} + +func NewIPBan(ban *IPBan) error { tx, err := BeginTx() if err != nil { return err } defer tx.Rollback() - stmt, err := PrepareSQL(query, tx) - if err != nil { - return err - } - defer stmt.Close() - if _, err = stmt.Exec( - ban.StaffID, ban.BoardID, ban.BannedForPostID, ban.CopyPostText, - ban.IsThreadBan, ban.IsActive, ban.RangeStart, ban.RangeEnd, ban.AppealAt, - ban.ExpiresAt, ban.Permanent, ban.StaffNote, ban.Message, ban.CanAppeal, - ); err != nil { - return err - } - ban.ID, err = getLatestID("DBPREFIXip_ban", tx) - if err != nil { + + if err = NewIPBanTx(tx, ban); err != nil { return err } + return tx.Commit() } From a34a1f1d573db4a0cae5746bc33ff6910bf6f3f3 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Fri, 3 Jan 2025 23:39:45 -0800 Subject: [PATCH 045/122] Add board ban migration and verify that no invalid bans were migrated --- cmd/gochan-migration/internal/pre2021/bans.go | 62 +++++++++++------- .../internal/pre2021/bans_test.go | 10 +++ tools/gochan-pre2021.sqlite3db | Bin 81920 -> 81920 bytes 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/bans.go b/cmd/gochan-migration/internal/pre2021/bans.go index a7dafb47..e5d15b19 100644 --- a/cmd/gochan-migration/internal/pre2021/bans.go +++ b/cmd/gochan-migration/internal/pre2021/bans.go @@ -1,6 +1,7 @@ package pre2021 import ( + "database/sql" "errors" "net" "strings" @@ -9,6 +10,7 @@ import ( "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/gcsql" "github.com/gochan-org/gochan/pkg/gcutil" + "github.com/rs/zerolog" ) type migrationBan struct { @@ -40,6 +42,27 @@ func (m *Pre2021Migrator) migrateBansInPlace() error { return common.NewMigrationError("pre2021", "migrateBansInPlace not yet implemented") } +func (m *Pre2021Migrator) migrateBan(tx *sql.Tx, ban *migrationBan, boardID *int, errEv *zerolog.Event) error { + migratedBan := &gcsql.IPBan{ + BoardID: boardID, + RangeStart: ban.ip, + RangeEnd: ban.ip, + IssuedAt: ban.timestamp, + } + migratedBan.CanAppeal = ban.canAppeal + migratedBan.AppealAt = ban.appealAt + migratedBan.ExpiresAt = ban.expires + migratedBan.Permanent = ban.permaban + migratedBan.Message = ban.reason + migratedBan.StaffNote = ban.staffNote + if err := gcsql.NewIPBanTx(tx, migratedBan); err != nil { + errEv.Err(err).Caller(). + Int("oldID", ban.oldID).Msg("Failed to migrate ban") + return err + } + return nil +} + func (m *Pre2021Migrator) migrateBansToNewDB() error { errEv := common.LogError() defer errEv.Discard() @@ -86,34 +109,29 @@ func (m *Pre2021Migrator) migrateBansToNewDB() error { } } - if len(ban.boardIDs) == 0 { - if ban.ip != "" { - if net.ParseIP(ban.ip) == nil { - gcutil.LogWarning(). - Int("oldID", ban.oldID). - Str("ip", ban.ip). - Msg("Found ban with invalid IP address, skipping") - continue + if ban.ip != "" { + if net.ParseIP(ban.ip) == nil { + gcutil.LogWarning(). + Int("oldID", ban.oldID). + Str("ip", ban.ip). + Msg("Found ban with invalid IP address, skipping") + continue + } + if len(ban.boardIDs) == 0 { + if err = m.migrateBan(tx, &ban, nil, errEv); err != nil { + return err } - migratedBan := &gcsql.IPBan{ - BoardID: nil, - RangeStart: ban.ip, - RangeEnd: ban.ip, - IssuedAt: ban.timestamp, + } else { + for b := range ban.boardIDs { + if err = m.migrateBan(tx, &ban, &ban.boardIDs[b], errEv); err != nil { + return err + } } - migratedBan.CanAppeal = ban.canAppeal - migratedBan.AppealAt = ban.appealAt - migratedBan.ExpiresAt = ban.expires - migratedBan.Permanent = ban.permaban - migratedBan.Message = ban.reason - migratedBan.StaffNote = ban.staffNote - gcsql.NewIPBanTx(tx, migratedBan) } } - } - return nil + return tx.Commit() } func (m *Pre2021Migrator) MigrateBans() error { diff --git a/cmd/gochan-migration/internal/pre2021/bans_test.go b/cmd/gochan-migration/internal/pre2021/bans_test.go index 1662e354..5c4f7cb3 100644 --- a/cmd/gochan-migration/internal/pre2021/bans_test.go +++ b/cmd/gochan-migration/internal/pre2021/bans_test.go @@ -3,6 +3,7 @@ package pre2021 import ( "testing" + "github.com/gochan-org/gochan/pkg/gcsql" "github.com/stretchr/testify/assert" ) @@ -24,4 +25,13 @@ func TestMigrateBansToNewDB(t *testing.T) { if !assert.NoError(t, migrator.MigrateBans()) { t.FailNow() } + bans, err := gcsql.GetIPBans(0, 200, false) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, 4, len(bans), "Expected to have 4 valid bans") + + var numInvalidBans int + assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXip_ban WHERE message = ?", []any{"Full ban on 8.8.0.0/16"}, []any{&numInvalidBans})) + assert.Equal(t, 0, numInvalidBans, "Expected the invalid test to not be migrated") } diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db index a78f7c80315e33f8a710fb5800aa3aafc949b278..555ae70da0fa2f788c00c29fd763db2cb55199b6 100644 GIT binary patch delta 82 zcmV-Y0ImOkfCYen1&|v7B#|6L0VJ_tp+5-@YXAid^Z^X?vkE}!3>tw30TKWS7Yzsv o4GkAFGYL5XK{E+FZfR^jlYmZelYUL@1ONa4gpq-Ov+qt3@qfD)jsO4v delta 76 zcmV-S0JHyqfCYen1&|v7Bas|K0VA Date: Sun, 5 Jan 2025 13:57:44 -0800 Subject: [PATCH 046/122] Make function for registering components and Post.NextFileOrder public --- pkg/gcsql/provisioning.go | 2 +- pkg/gcsql/uploads.go | 6 ++++-- pkg/gcsql/util.go | 15 ++++++++++----- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pkg/gcsql/provisioning.go b/pkg/gcsql/provisioning.go index cd9591ba..b1802f0f 100644 --- a/pkg/gcsql/provisioning.go +++ b/pkg/gcsql/provisioning.go @@ -54,7 +54,7 @@ func GetCompleteDatabaseVersion() (dbVersion, dbFlag int, err error) { return 0, 0, err } if versionTableExists { - databaseVersion, versionError := getDatabaseVersion(gochanVersionKeyConstant) + databaseVersion, versionError := GetComponentVersion(gochanVersionKeyConstant) if versionError != nil { return 0, 0, versionError } diff --git a/pkg/gcsql/uploads.go b/pkg/gcsql/uploads.go index aee41172..dc06a329 100644 --- a/pkg/gcsql/uploads.go +++ b/pkg/gcsql/uploads.go @@ -1,6 +1,7 @@ package gcsql import ( + "context" "database/sql" "errors" @@ -42,10 +43,11 @@ func GetThreadFiles(post *Post) ([]Upload, error) { return uploads, nil } -func (p *Post) nextFileOrder() (int, error) { +// NextFileOrder gets what would be the next file_order value (not particularly useful until multi-file posting is implemented) +func (p *Post) NextFileOrder(ctx context.Context, tx *sql.Tx) (int, error) { const query = `SELECT COALESCE(MAX(file_order) + 1, 0) FROM DBPREFIXfiles WHERE post_id = ?` var next int - err := QueryRowSQL(query, []any{p.ID}, []any{&next}) + err := QueryRowContextSQL(ctx, tx, query, []any{p.ID}, []any{&next}) return next, err } diff --git a/pkg/gcsql/util.go b/pkg/gcsql/util.go index bee04275..872b286d 100644 --- a/pkg/gcsql/util.go +++ b/pkg/gcsql/util.go @@ -400,17 +400,22 @@ func doesTableExist(tableName string) (bool, error) { return count > 0, nil } -// getDatabaseVersion gets the version of the database, or an error if none or multiple exist -func getDatabaseVersion(componentKey string) (int, error) { +// GetComponentVersion gets the version of the database component (e.g., gochan), or an error if none exist +func GetComponentVersion(componentKey string) (int, error) { const sql = `SELECT version FROM DBPREFIXdatabase_version WHERE component = ?` var version int err := QueryRowSQL(sql, []any{componentKey}, []any{&version}) - if err != nil { - return 0, err - } return version, err } +// RegisterComponent adds a new component and version to the database_version table. It returns an error if +// the component is already in the table, or any other SQL errors that occurred +func RegisterComponent(tx *sql.Tx, component string, version int) error { + const sql = "INSERT INTO DBPREFIXdatabase_version (component, version) VALUES (?,?)" + _, err := ExecTxSQL(tx, sql, component, version) + return err +} + // doesGochanPrefixTableExist returns true if any table with a gochan prefix was found. // Returns false if the prefix is an empty string func doesGochanPrefixTableExist() (bool, error) { From b0635b15f6519c5f8a0cd194134170b2640bfd10 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 5 Jan 2025 13:58:48 -0800 Subject: [PATCH 047/122] Organize .gitignore, add stuff to SQLite test db --- .gitignore | 35 ++++++++++++++++++++------------- tools/gochan-pre2021.sqlite3db | Bin 81920 -> 81920 bytes 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index fc334aa7..cce8d946 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,30 @@ -/gochan* -/lib/ -/log/ -/releases/ -/vagrant/.vagrant/ -/html/boards.json +*.bak /html/index.html /html/test* -/html/js/ -/templates/override -*.bak -*.log +/html/boards.json +.vscode/settings.json +.parcel-cache *.swp -*.db +.vagrant/ +*.log +/templates/override + +# Go output +/gochan* +/releases/ *.so -__debug_bin +__debug_bin* + +# Node.js/TypeScript +/html/js/ node_modules /frontend/coverage /frontend/tests/coverage -.parcel-cache + +# Python __pycache__ .venv/ -.vscode/settings.json \ No newline at end of file + +# SQLite +*.db +*.*-journal \ No newline at end of file diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db index 555ae70da0fa2f788c00c29fd763db2cb55199b6..99c79a9637ffb910a3ca295ab31975437b14399e 100644 GIT binary patch delta 165 zcmZo@U~On%ogmGqI#I@%QFUX&5`H#D{v8bbJ2ndntm98|V`O76H5E5@%*)F!%}Y+r zP0cG&C@xLPN=+_75z9?2E>28OO-#wn%ri1DGSxLO)HN_wFfg++G_f)? Date: Sun, 5 Jan 2025 14:11:11 -0800 Subject: [PATCH 048/122] Add stubs for pre-2021 staff migration --- .../internal/common/handler.go | 22 ++++++------ .../internal/gcupdate/filters.go | 1 - .../internal/gcupdate/gcupdate.go | 2 +- .../internal/pre2021/boards.go | 4 ++- .../internal/pre2021/pre2021.go | 15 ++++---- .../internal/pre2021/staff.go | 36 +++++++++++++++++++ .../internal/pre2021/staff_test.go | 23 ++++++++++++ 7 files changed, 80 insertions(+), 23 deletions(-) create mode 100644 cmd/gochan-migration/internal/pre2021/staff.go create mode 100644 cmd/gochan-migration/internal/pre2021/staff_test.go diff --git a/cmd/gochan-migration/internal/common/handler.go b/cmd/gochan-migration/internal/common/handler.go index b997f81d..d26d2a48 100644 --- a/cmd/gochan-migration/internal/common/handler.go +++ b/cmd/gochan-migration/internal/common/handler.go @@ -69,28 +69,26 @@ type DBMigrator interface { // logging any errors that occur during the migration MigrateDB() (bool, error) - // MigrateBoards gets info about the old boards in the board table and inserts each one - // into the new database if they don't already exist + // MigrateBoards migrates the board sections (if they exist) and boards if each one + // doesn't already exists MigrateBoards() error - // MigratePosts gets the threads and replies in the old database, and inserts them into - // the new database, creating new threads to avoid putting replies in threads that already - // exist + // MigratePosts gets the threads and replies (excluding deleted ones) in the old database, and inserts them into + // the new database, creating new threads to avoid putting replies in threads that already exist MigratePosts() error // MigrateStaff gets the staff list in the old board and inserts them into the new board if - // the username doesn't already exist. It sets the starting password to the given password - MigrateStaff(password string) error + // the username doesn't already exist. Migrated staff accounts will need to have their password reset + // in order to be logged into + MigrateStaff() error - // MigrateBans gets the list of bans and appeals in the old database and inserts them into the - // new one if, for each entry, the IP/name/etc isn't already banned for the same length - // e.g. 1.1.1.1 is permabanned on both, 1.1.2.2 is banned for 5 days on both, etc + // MigrateBans migrates IP bans, appeals, and filters MigrateBans() error // MigrateAnnouncements gets the list of public and staff announcements in the old database - // and inserts them into the new database, + // and inserts them into the new database MigrateAnnouncements() error - // Close closes the database if initialized and deltes the temporary columns created + // Close closes the database if initialized and deletes any temporary columns created Close() error } diff --git a/cmd/gochan-migration/internal/gcupdate/filters.go b/cmd/gochan-migration/internal/gcupdate/filters.go index 322d6e34..76a25137 100644 --- a/cmd/gochan-migration/internal/gcupdate/filters.go +++ b/cmd/gochan-migration/internal/gcupdate/filters.go @@ -32,7 +32,6 @@ type filenameOrUsernameBanBase struct { StaffID int // sql: staff_id StaffNote string // sql: staff_note IssuedAt time.Time // sql: issued_at - check string // replaced with username or filename IsRegex bool // sql: is_regex } diff --git a/cmd/gochan-migration/internal/gcupdate/gcupdate.go b/cmd/gochan-migration/internal/gcupdate/gcupdate.go index 7e194781..1059cc40 100644 --- a/cmd/gochan-migration/internal/gcupdate/gcupdate.go +++ b/cmd/gochan-migration/internal/gcupdate/gcupdate.go @@ -475,7 +475,7 @@ func (*GCDatabaseUpdater) MigratePosts() error { return gcutil.ErrNotImplemented } -func (*GCDatabaseUpdater) MigrateStaff(_ string) error { +func (*GCDatabaseUpdater) MigrateStaff() error { return gcutil.ErrNotImplemented } diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index a7b842f3..b16b2981 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -22,7 +22,9 @@ type migrationSection struct { } func (m *Pre2021Migrator) migrateSectionsInPlace() error { - return common.NewMigrationError("pre2021", "migrateSectionsInPlace not implemented") + err := common.NewMigrationError("pre2021", "migrateSectionsInPlace not implemented") + common.LogError().Err(err).Caller().Msg("Failed to migrate sections") + return err } func (m *Pre2021Migrator) migrateBoardsInPlace() error { diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index 7d80e3a3..b5d1f78b 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -83,12 +83,15 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) { return false, err } common.LogInfo().Msg("Migrated boards successfully") + if err = m.MigratePosts(); err != nil { return false, err } - // if err = m.MigrateStaff("password"); err != nil { - // return false, err - // } + common.LogInfo().Msg("Migrated threads, posts, and uploads successfully") + + if err = m.MigrateStaff(); err != nil { + return false, err + } if err = m.MigrateBans(); err != nil { return false, err } @@ -99,12 +102,8 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) { return true, nil } -func (*Pre2021Migrator) MigrateStaff(_ string) error { - return nil -} - func (*Pre2021Migrator) MigrateAnnouncements() error { - return nil + return common.NewMigrationError("pre2021", "MigrateAnnouncements not yet implemented") } func (m *Pre2021Migrator) Close() error { diff --git a/cmd/gochan-migration/internal/pre2021/staff.go b/cmd/gochan-migration/internal/pre2021/staff.go new file mode 100644 index 00000000..4ccc754a --- /dev/null +++ b/cmd/gochan-migration/internal/pre2021/staff.go @@ -0,0 +1,36 @@ +package pre2021 + +import ( + "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" + "github.com/gochan-org/gochan/pkg/gcsql" +) + +type migrationStaff struct { + gcsql.Staff + + oldID int + boardIDs []int +} + +func (*Pre2021Migrator) migrateStaffInPlace() error { + err := common.NewMigrationError("pre2021", "migrateSectionsInPlace not yet implemented") + common.LogError().Err(err).Caller().Msg("Failed to migrate sections") + return err +} + +func (*Pre2021Migrator) migrateStaffToNewDB() error { + errEv := common.LogError() + defer errEv.Discard() + + err := common.NewMigrationError("pre2021", "migrateStaffToNewDB not yet implemented") + errEv.Err(err).Caller().Msg("Failed to migrate sections") + + return err +} + +func (m *Pre2021Migrator) MigrateStaff() error { + if m.IsMigratingInPlace() { + return m.migrateStaffInPlace() + } + return m.migrateStaffToNewDB() +} diff --git a/cmd/gochan-migration/internal/pre2021/staff_test.go b/cmd/gochan-migration/internal/pre2021/staff_test.go new file mode 100644 index 00000000..bfbbf348 --- /dev/null +++ b/cmd/gochan-migration/internal/pre2021/staff_test.go @@ -0,0 +1,23 @@ +package pre2021 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMigrateStaffToNewDB(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, false) + if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateBoards()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateStaff()) { + t.FailNow() + } +} From 7cb3100140db60de88b641ea8558afbeb653f6ac Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 5 Jan 2025 15:18:11 -0800 Subject: [PATCH 049/122] Implement pre-2021 staff migration to new DB --- .../internal/pre2021/bans_test.go | 2 +- .../internal/pre2021/posts.go | 5 + .../internal/pre2021/pre2021.go | 5 +- .../internal/pre2021/queries.go | 2 + .../internal/pre2021/staff.go | 116 ++++++++++++++++-- .../internal/pre2021/staff_test.go | 6 + tools/gochan-pre2021.sqlite3db | Bin 81920 -> 81920 bytes 7 files changed, 122 insertions(+), 14 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/bans_test.go b/cmd/gochan-migration/internal/pre2021/bans_test.go index 5c4f7cb3..5f9e2c16 100644 --- a/cmd/gochan-migration/internal/pre2021/bans_test.go +++ b/cmd/gochan-migration/internal/pre2021/bans_test.go @@ -29,7 +29,7 @@ func TestMigrateBansToNewDB(t *testing.T) { if !assert.NoError(t, err) { t.FailNow() } - assert.Equal(t, 4, len(bans), "Expected to have 4 valid bans") + assert.Equal(t, 6, len(bans), "Expected to have 4 valid bans") var numInvalidBans int assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXip_ban WHERE message = ?", []any{"Full ban on 8.8.0.0/16"}, []any{&numInvalidBans})) diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index b10e5bd0..1f786a3b 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -171,6 +171,11 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { } migratedThreads++ } + if err = rows.Close(); err != nil { + errEv.Err(err).Caller().Msg("Failed to close posts rows") + return err + } + if len(threadIDsWithInvalidBoards) > 0 { errEv.Caller(). Ints("threadIDs", threadIDsWithInvalidBoards). diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index b5d1f78b..105c14a0 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -22,8 +22,9 @@ type Pre2021Migrator struct { options *common.MigrationOptions config Pre2021Config - boards []migrationBoard - sections []migrationSection + migrationUser *gcsql.Staff + boards []migrationBoard + sections []migrationSection } // IsMigratingInPlace implements common.DBMigrator. diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index 12069581..526a00a1 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -14,6 +14,8 @@ bumped, stickied, locked FROM DBPREFIXposts WHERE deleted_timestamp IS NULL` threadsQuery = postsQuery + " AND parentid = 0" + staffQuery = `SELECT username, rank, boards, added_on, last_active FROM DBPREFIXstaff` + bansQuery = `SELECT id, allow_read, COALESCE(ip, '') as ip, name, name_is_regex, filename, file_checksum, boards, staff, timestamp, expires, permaban, reason, type, staff_note, appeal_at, can_appeal FROM DBPREFIXbanlist` ) diff --git a/cmd/gochan-migration/internal/pre2021/staff.go b/cmd/gochan-migration/internal/pre2021/staff.go index 4ccc754a..d2fb79df 100644 --- a/cmd/gochan-migration/internal/pre2021/staff.go +++ b/cmd/gochan-migration/internal/pre2021/staff.go @@ -1,31 +1,125 @@ package pre2021 import ( + "errors" + "strings" + "time" + "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/gcsql" + "github.com/gochan-org/gochan/pkg/gcutil" + "github.com/rs/zerolog" ) -type migrationStaff struct { - gcsql.Staff - - oldID int - boardIDs []int -} - func (*Pre2021Migrator) migrateStaffInPlace() error { err := common.NewMigrationError("pre2021", "migrateSectionsInPlace not yet implemented") common.LogError().Err(err).Caller().Msg("Failed to migrate sections") return err } -func (*Pre2021Migrator) migrateStaffToNewDB() error { +func (m *Pre2021Migrator) getMigrationUser(errEv *zerolog.Event) (*gcsql.Staff, error) { + if m.migrationUser != nil { + return m.migrationUser, nil + } + + user := &gcsql.Staff{ + Username: "pre2021-migration" + gcutil.RandomString(15), + AddedOn: time.Now(), + } + _, err := gcsql.ExecSQL("INSERT INTO DBPREFIXstaff(username,password_checksum,global_rank,is_active) values(?,'',0,0)", user.Username) + if err != nil { + errEv.Err(err).Caller().Str("username", user.Username).Msg("Failed to create migration user") + return nil, err + } + + if err = gcsql.QueryRowSQL("SELECT id FROM DBPREFIXstaff WHERE username = ?", []any{user.Username}, []any{&user.ID}); err != nil { + errEv.Err(err).Caller().Str("username", user.Username).Msg("Failed to get migration user ID") + return nil, err + } + m.migrationUser = user + return user, nil +} + +func (m *Pre2021Migrator) migrateStaffToNewDB() error { errEv := common.LogError() defer errEv.Discard() - err := common.NewMigrationError("pre2021", "migrateStaffToNewDB not yet implemented") - errEv.Err(err).Caller().Msg("Failed to migrate sections") + _, err := m.getMigrationUser(errEv) + if err != nil { + return err + } - return err + rows, err := m.db.QuerySQL(staffQuery) + if err != nil { + errEv.Err(err).Caller().Msg("Failed to get ban rows") + return err + } + defer rows.Close() + + for rows.Next() { + var username string + var rank int + var boards string + var addedOn, lastActive time.Time + + if err = rows.Scan(&username, &rank, &boards, &addedOn, &lastActive); err != nil { + errEv.Err(err).Caller().Msg("Failed to scan staff row") + return err + } + _, err = gcsql.GetStaffByUsername(username, false) + if err == nil { + // found staff + gcutil.LogInfo().Str("username", username).Int("rank", rank).Msg("Found matching staff account") + } + if errors.Is(err, gcsql.ErrUnrecognizedUsername) { + // staff doesn't exist, create it (with invalid checksum to be updated by the admin) + if _, err2 := gcsql.ExecSQL( + "INSERT INTO DBPREFIXstaff(username,password_checksum,global_rank,added_on,last_login,is_active) values(?,'',?,?,?,1)", + username, rank, addedOn, lastActive, + ); err2 != nil { + errEv.Err(err2).Caller(). + Str("username", username).Int("rank", rank). + Msg("Failed to migrate staff account") + return err + } + gcutil.LogInfo().Str("username", username).Int("rank", rank).Msg("Successfully migrated staff account") + } else if err != nil { + errEv.Err(err).Caller().Str("username", username).Msg("Failed to get staff account info") + return err + } + staffID, err := gcsql.GetStaffID(username) + if err != nil { + errEv.Err(err).Caller().Str("username", username).Msg("Failed to get staff account ID") + return err + } + if boards != "" && boards != "*" { + boardsArr := strings.Split(boards, ",") + for _, board := range boardsArr { + board = strings.TrimSpace(board) + boardID, err := gcsql.GetBoardIDFromDir(board) + if err != nil { + errEv.Err(err).Caller(). + Str("username", username). + Str("board", board). + Msg("Failed to get board ID") + return err + } + if _, err = gcsql.ExecSQL("INSERT INTO DBPREFIXboard_staff(board_id,staff_id) VALUES(?,?)", boardID, staffID); err != nil { + errEv.Err(err).Caller(). + Str("username", username). + Str("board", board). + Msg("Failed to apply staff board info") + return err + } + } + } + } + + if err = rows.Close(); err != nil { + errEv.Err(err).Caller().Msg("Failed to close staff rows") + return err + } + return nil } func (m *Pre2021Migrator) MigrateStaff() error { diff --git a/cmd/gochan-migration/internal/pre2021/staff_test.go b/cmd/gochan-migration/internal/pre2021/staff_test.go index bfbbf348..5c81dea3 100644 --- a/cmd/gochan-migration/internal/pre2021/staff_test.go +++ b/cmd/gochan-migration/internal/pre2021/staff_test.go @@ -3,6 +3,7 @@ package pre2021 import ( "testing" + "github.com/gochan-org/gochan/pkg/gcsql" "github.com/stretchr/testify/assert" ) @@ -20,4 +21,9 @@ func TestMigrateStaffToNewDB(t *testing.T) { if !assert.NoError(t, migrator.MigrateStaff()) { t.FailNow() } + migratedAdmin, err := gcsql.GetStaffByUsername("migratedadmin", true) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, 3, migratedAdmin.Rank) } diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db index 99c79a9637ffb910a3ca295ab31975437b14399e..04ab152fb15a32b7ac781003429af6abedf74943 100644 GIT binary patch delta 634 zcmZo@U~On%ogmGqF;T{uQDbAmLV0ffM+}_&@eKU$`Qtb9C%Y9sQTnTekL5bndwD|C8;Ts4gAxU8u=K3(&D@vyu7^X#>SkMjE=^fHhGyj zc1X%ZCM)?yTk`cYh`{WG(T&`!K<&~X?ff9^iHy9)92RdV+bjtA*d1{p7jtjp9)&!{A|xI`x- zGbJT84`y3pN^WMJk%5t^u7RPhfvJL#p_Q?*m7y^fp`g^n;`}^jEHZh_VAm?b{H4f= z;#?!Rb2o4I_dUSD#GlE)znedEV`D6TN&piFgRUf_vavDDNlE#MMJc)YDTztRhDOFr zNRCFf+Stm#!pgu9i;y!T2ZOpKBR|5#>1X8`&4Je50;QH)n*|fj@hgfj>oTI-$Rxn5 O&IuFT{#&1sRR93NY?sOa delta 308 zcmZo@U~On%ogmGqI#I@%QFUX&LV0cm1_n<4Tn7I4{Es&CC%oIG)S5sB%R2}Ys_Jx2ZmacxA|r&q_F@M zD8m)V0~K%@S{msYnpx->>KSWIcK7pUY?;{Dz}u+G1XQXj3q+#glRbS*Cb#;eE2J@_ z=vU-K(Ql-+dAq;w0X9bd!wmd~Hwz~0oc+n000QCMQZ>6 From 01d7a722a5553691b9f6fdf951730899d9a789b3 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 5 Jan 2025 15:37:46 -0800 Subject: [PATCH 050/122] Properly migrate ban staff ID --- cmd/gochan-migration/internal/pre2021/bans.go | 15 +++++++++++++++ .../internal/pre2021/bans_test.go | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/cmd/gochan-migration/internal/pre2021/bans.go b/cmd/gochan-migration/internal/pre2021/bans.go index e5d15b19..e00ccb63 100644 --- a/cmd/gochan-migration/internal/pre2021/bans.go +++ b/cmd/gochan-migration/internal/pre2021/bans.go @@ -54,6 +54,7 @@ func (m *Pre2021Migrator) migrateBan(tx *sql.Tx, ban *migrationBan, boardID *int migratedBan.ExpiresAt = ban.expires migratedBan.Permanent = ban.permaban migratedBan.Message = ban.reason + migratedBan.StaffID = ban.staffID migratedBan.StaffNote = ban.staffNote if err := gcsql.NewIPBanTx(tx, migratedBan); err != nil { errEv.Err(err).Caller(). @@ -109,6 +110,20 @@ func (m *Pre2021Migrator) migrateBansToNewDB() error { } } + ban.staffID, err = gcsql.GetStaffID(ban.staff) + if errors.Is(err, gcsql.ErrUnrecognizedUsername) { + // username not found after staff were migrated, use a stand-in account to be updated by the admin later + migrationUser, err := m.getMigrationUser(errEv) + if err != nil { + return err + } + common.LogWarning(). + Str("username", ban.staff). + Str("migrationUser", migrationUser.Username). + Msg("Ban staff not found in migrated staff table, using migration user instead") + ban.staffID = migrationUser.ID + } + if ban.ip != "" { if net.ParseIP(ban.ip) == nil { gcutil.LogWarning(). diff --git a/cmd/gochan-migration/internal/pre2021/bans_test.go b/cmd/gochan-migration/internal/pre2021/bans_test.go index 5f9e2c16..01add7af 100644 --- a/cmd/gochan-migration/internal/pre2021/bans_test.go +++ b/cmd/gochan-migration/internal/pre2021/bans_test.go @@ -22,6 +22,10 @@ func TestMigrateBansToNewDB(t *testing.T) { t.FailNow() } + if !assert.NoError(t, migrator.MigrateStaff()) { + t.FailNow() + } + if !assert.NoError(t, migrator.MigrateBans()) { t.FailNow() } @@ -30,6 +34,7 @@ func TestMigrateBansToNewDB(t *testing.T) { t.FailNow() } assert.Equal(t, 6, len(bans), "Expected to have 4 valid bans") + assert.NotZero(t, bans[0].StaffID, "Expected ban staff ID field to be set") var numInvalidBans int assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXip_ban WHERE message = ?", []any{"Full ban on 8.8.0.0/16"}, []any{&numInvalidBans})) From 4f9f1e0d3fad7339e9d81a6968052743e29cbedb Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 5 Jan 2025 16:01:17 -0800 Subject: [PATCH 051/122] Add migrating pre-2021 name/file bans to filters --- cmd/gochan-migration/internal/pre2021/bans.go | 62 +++++++++++++++++-- .../internal/pre2021/bans_test.go | 11 ++++ 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/bans.go b/cmd/gochan-migration/internal/pre2021/bans.go index e00ccb63..1fbfdfdb 100644 --- a/cmd/gochan-migration/internal/pre2021/bans.go +++ b/cmd/gochan-migration/internal/pre2021/bans.go @@ -1,6 +1,7 @@ package pre2021 import ( + "context" "database/sql" "errors" "net" @@ -33,9 +34,7 @@ type migrationBan struct { canAppeal bool boardIDs []int - banID int staffID int - filterID int } func (m *Pre2021Migrator) migrateBansInPlace() error { @@ -110,20 +109,27 @@ func (m *Pre2021Migrator) migrateBansToNewDB() error { } } + migrationUser, err := m.getMigrationUser(errEv) + if err != nil { + return err + } ban.staffID, err = gcsql.GetStaffID(ban.staff) if errors.Is(err, gcsql.ErrUnrecognizedUsername) { // username not found after staff were migrated, use a stand-in account to be updated by the admin later - migrationUser, err := m.getMigrationUser(errEv) - if err != nil { - return err - } common.LogWarning(). Str("username", ban.staff). Str("migrationUser", migrationUser.Username). Msg("Ban staff not found in migrated staff table, using migration user instead") ban.staffID = migrationUser.ID + } else if err != nil { + errEv.Err(err).Caller().Str("username", ban.staff).Msg("Failed to get staff from username") + return err } + if ban.ip == "" && ban.name == "" && ban.fileChecksum == "" && ban.filename == "" { + common.LogWarning().Int("banID", ban.oldID).Msg("Found invalid ban (no IP, name, file checksum, or filename set)") + continue + } if ban.ip != "" { if net.ParseIP(ban.ip) == nil { gcutil.LogWarning(). @@ -144,6 +150,50 @@ func (m *Pre2021Migrator) migrateBansToNewDB() error { } } } + if ban.name != "" || ban.fileChecksum != "" || ban.filename != "" { + filter := &gcsql.Filter{ + StaffID: &ban.staffID, + StaffNote: ban.staffNote, + IsActive: true, + HandleIfAny: true, + MatchAction: "reject", + MatchDetail: ban.reason, + } + var conditions []gcsql.FilterCondition + if ban.name != "" { + nameCondition := gcsql.FilterCondition{ + Field: "name", + Search: ban.name, + MatchMode: gcsql.ExactMatch, + } + if ban.nameIsRegex { + nameCondition.MatchMode = gcsql.RegexMatch + } + conditions = append(conditions, nameCondition) + } + if ban.fileChecksum != "" { + conditions = append(conditions, gcsql.FilterCondition{ + Field: "checksum", + MatchMode: gcsql.ExactMatch, + Search: ban.fileChecksum, + }) + } + if ban.filename != "" { + filenameCondition := gcsql.FilterCondition{ + Field: "filename", + Search: ban.filename, + MatchMode: gcsql.ExactMatch, + } + if ban.nameIsRegex { + filenameCondition.MatchMode = gcsql.RegexMatch + } + conditions = append(conditions, filenameCondition) + } + if err = gcsql.ApplyFilterTx(context.Background(), tx, filter, conditions, ban.boardIDs); err != nil { + errEv.Err(err).Caller().Int("banID", ban.oldID).Msg("Failed to migrate ban to filter") + return err + } + } } return tx.Commit() diff --git a/cmd/gochan-migration/internal/pre2021/bans_test.go b/cmd/gochan-migration/internal/pre2021/bans_test.go index 01add7af..f527894c 100644 --- a/cmd/gochan-migration/internal/pre2021/bans_test.go +++ b/cmd/gochan-migration/internal/pre2021/bans_test.go @@ -39,4 +39,15 @@ func TestMigrateBansToNewDB(t *testing.T) { var numInvalidBans int assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXip_ban WHERE message = ?", []any{"Full ban on 8.8.0.0/16"}, []any{&numInvalidBans})) assert.Equal(t, 0, numInvalidBans, "Expected the invalid test to not be migrated") + + filters, err := gcsql.GetAllFilters(gcsql.TrueOrFalse) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, 1, len(filters)) + conditions, err := filters[0].Conditions() + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, 3, len(conditions), "Expected filter to have three conditions") } From 175710b43b3ea693e2e158658d7db7eeaa4fa1f1 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 5 Jan 2025 17:23:18 -0800 Subject: [PATCH 052/122] Add announcement migration --- .../internal/pre2021/announcements.go | 66 ++++++++++++++++++ .../internal/pre2021/pre2021.go | 15 ++-- .../internal/pre2021/pre2021_test.go | 11 +++ .../internal/pre2021/queries.go | 2 + tools/gochan-pre2021.sqlite3db | Bin 81920 -> 81920 bytes 5 files changed, 87 insertions(+), 7 deletions(-) create mode 100644 cmd/gochan-migration/internal/pre2021/announcements.go diff --git a/cmd/gochan-migration/internal/pre2021/announcements.go b/cmd/gochan-migration/internal/pre2021/announcements.go new file mode 100644 index 00000000..e885e296 --- /dev/null +++ b/cmd/gochan-migration/internal/pre2021/announcements.go @@ -0,0 +1,66 @@ +package pre2021 + +import ( + "errors" + "time" + + "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" + "github.com/gochan-org/gochan/pkg/gcsql" +) + +func (*Pre2021Migrator) migrateAnnouncementsInPlace() error { + return common.NewMigrationError("pre2021", "migrateAnnouncementsInPlace not implemented") +} + +func (m *Pre2021Migrator) migrateAnnouncementsToNewDB() error { + errEv := common.LogError() + rows, err := m.db.QuerySQL(announcementsQuery) + if err != nil { + errEv.Err(err).Caller().Msg("Failed to get announcements") + return err + } + defer rows.Close() + + if _, err = m.getMigrationUser(errEv); err != nil { + return err + } + + for rows.Next() { + var id int + var subject, message, staff string + var timestamp time.Time + if err = rows.Scan(&id, &subject, &message, &staff, ×tamp); err != nil { + errEv.Err(err).Caller().Msg("Failed to scan announcement row") + return err + } + staffID, err := gcsql.GetStaffID(staff) + if errors.Is(err, gcsql.ErrUnrecognizedUsername) { + // user doesn't exist, use migration user + common.LogWarning().Str("staff", staff).Msg("Staff username not found in database") + message += "\n(originally by " + staff + ")" + } else if err != nil { + errEv.Err(err).Caller().Str("staff", staff).Msg("Failed to get staff ID") + return err + } + if _, err = gcsql.ExecSQL( + "INSERT INTO DBPREFIXannouncements(staff_id,subject,message,timestamp) values(?,?,?,?)", + staffID, subject, message, timestamp, + ); err != nil { + errEv.Err(err).Caller().Str("staff", staff).Msg("Failed to migrate announcement") + return err + } + } + + if err = rows.Close(); err != nil { + errEv.Err(err).Caller().Msg("Failed to close announcement rows") + return err + } + return nil +} + +func (m *Pre2021Migrator) MigrateAnnouncements() error { + if m.IsMigratingInPlace() { + return m.migrateAnnouncementsInPlace() + } + return m.migrateAnnouncementsToNewDB() +} diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index 105c14a0..a4fe542b 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -93,20 +93,21 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) { if err = m.MigrateStaff(); err != nil { return false, err } + common.LogInfo().Msg("Migrated staff successfully") + if err = m.MigrateBans(); err != nil { return false, err } - // if err = m.MigrateAnnouncements(); err != nil { - // return false, err - // } + common.LogInfo().Msg("Migrated bans and filters successfully") + + if err = m.MigrateAnnouncements(); err != nil { + return false, err + } + common.LogInfo().Msg("Migrated staff announcements successfully") return true, nil } -func (*Pre2021Migrator) MigrateAnnouncements() error { - return common.NewMigrationError("pre2021", "MigrateAnnouncements not yet implemented") -} - func (m *Pre2021Migrator) Close() error { if m.db != nil { return m.db.Close() diff --git a/cmd/gochan-migration/internal/pre2021/pre2021_test.go b/cmd/gochan-migration/internal/pre2021/pre2021_test.go index e28eb26c..71a651b9 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021_test.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021_test.go @@ -89,3 +89,14 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 return migrator } + +func TestPre2021Migration(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, false) + if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { + t.FailNow() + } + migrated, err := migrator.MigrateDB() + assert.True(t, migrated) + assert.NoError(t, err) +} diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index 526a00a1..b8fbbc68 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -18,4 +18,6 @@ bumped, stickied, locked FROM DBPREFIXposts WHERE deleted_timestamp IS NULL` bansQuery = `SELECT id, allow_read, COALESCE(ip, '') as ip, name, name_is_regex, filename, file_checksum, boards, staff, timestamp, expires, permaban, reason, type, staff_note, appeal_at, can_appeal FROM DBPREFIXbanlist` + + announcementsQuery = "SELECT id, subject, message, poster, timestamp FROM DBPREFIXannouncements" ) diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db index 04ab152fb15a32b7ac781003429af6abedf74943..9c99c2aba1ac978c25bc47d4d5982d02f3015c20 100644 GIT binary patch delta 54 zcmZo@U~On%ogmGqIZ?)$QFCL$GJci~4E!553kodbXZK@dV=y&UpS)jQbF#gkSo1ge K?cd}XxfB4go)8EC delta 50 zcmZo@U~On%ogmGqF;T{uQDbAmGJcjF4E#Gb3nr}NXLn;{V=y%ppS)dOv-ySm_80Pu G5()r)Dh~<( From ea20cc408f638664376b1c0b18b5d997aeac33cd Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 5 Jan 2025 17:28:48 -0800 Subject: [PATCH 053/122] Add cmd directory to test --- build.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/build.py b/build.py index c5fbe2dd..e2c6feb6 100755 --- a/build.py +++ b/build.py @@ -425,15 +425,13 @@ def sass(watch=False): sys.exit(status) def test(verbose=False, coverage=False): - pkgs = os.listdir("pkg") - for pkg in pkgs: - cmd = ["go", "test"] - if verbose: - cmd += ["-v"] - if coverage: - cmd += ["-cover"] - cmd += [path.join("./pkg", pkg)] - run_cmd(cmd, realtime=True, print_command=True) + cmd = ["go", "test"] + if verbose: + cmd += ["-v"] + if coverage: + cmd += ["-cover"] + cmd += ["./pkg/...", "./cmd/..."] + run_cmd(cmd, realtime=True, print_command=True) if __name__ == "__main__": From 9e61ef2730c73a0778a84f7dfab39c338cc09938 Mon Sep 17 00:00:00 2001 From: onihilist Date: Tue, 7 Jan 2025 12:01:18 +0100 Subject: [PATCH 054/122] Add update board button --- templates/boardpage.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/boardpage.html b/templates/boardpage.html index 803b696c..55cfdcb0 100644 --- a/templates/boardpage.html +++ b/templates/boardpage.html @@ -34,6 +34,8 @@
+ Update + | Scroll to top
From ab42c1332b9471c259798b4b2950ec581a49855f Mon Sep 17 00:00:00 2001 From: onihilist Date: Tue, 7 Jan 2025 12:24:11 +0100 Subject: [PATCH 055/122] Add update thread button --- templates/threadpage.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/threadpage.html b/templates/threadpage.html index be856688..7d309126 100644 --- a/templates/threadpage.html +++ b/templates/threadpage.html @@ -3,7 +3,7 @@

/{{$.board.Dir}}/ - {{$.board.Title}}

{{$.board.Subtitle}}
- Return | Catalog | Bottom + Return | Update | Catalog | Bottom

{{template "postbox.html" .}}
From 1704bbe959da68eed2aa48c9cfa370dec3e1dd10 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 8 Jan 2025 15:03:10 -0800 Subject: [PATCH 056/122] Update failing board test cases --- pkg/gctemplates/templatetests/templatecases_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/gctemplates/templatetests/templatecases_test.go b/pkg/gctemplates/templatetests/templatecases_test.go index 740ed489..25f233bb 100644 --- a/pkg/gctemplates/templatetests/templatecases_test.go +++ b/pkg/gctemplates/templatetests/templatecases_test.go @@ -208,7 +208,7 @@ var ( }, }, expectedOutput: boardPageHeaderBase + - `

Report reason:
 
[1]
[home] []
` + + `

Report reason:
 
` + footer, }, { @@ -222,7 +222,7 @@ var ( }, }, expectedOutput: boardPageHeaderBase + - `

Report reason:
 
` + + `

Report reason:
 
` + footer, }, } From 22eff8d1d7c6f7d62c8b3575d5b1f638624d2566 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 8 Jan 2025 15:41:12 -0800 Subject: [PATCH 057/122] Update JS board events test --- frontend/tests/boardevents.test.ts | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/frontend/tests/boardevents.test.ts b/frontend/tests/boardevents.test.ts index 620f300a..79c32d14 100644 --- a/frontend/tests/boardevents.test.ts +++ b/frontend/tests/boardevents.test.ts @@ -9,43 +9,42 @@ import { applyBBCode, handleKeydown } from "../ts/boardevents"; document.documentElement.innerHTML = simpleHTML; -function doBBCode(keycode: number, text: string, start: number, end: number) { +function doBBCode(key:string, text: string, start: number, end: number) { const $ta = $("` + `File` + `` + - `Options` + + `Options` + `Password(for post/file deletion)` + `

` ) From 88f645e017c8ce19e9bc232809262397374aadda Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 19 Jan 2025 12:12:31 -0800 Subject: [PATCH 075/122] Fix Selenium cyclic thread test --- tools/selenium_testing/tests/test_posting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/selenium_testing/tests/test_posting.py b/tools/selenium_testing/tests/test_posting.py index d1be2f8c..12d6264d 100644 --- a/tools/selenium_testing/tests/test_posting.py +++ b/tools/selenium_testing/tests/test_posting.py @@ -101,6 +101,7 @@ class TestPosting(SeleniumTestCase): WebDriverWait(self.driver, 10).until( EC.element_to_be_clickable((By.CSS_SELECTOR, "form#postform input[type=submit]"))) form = self.driver.find_element(by=By.CSS_SELECTOR, value="form#postform") + form.find_element(by=By.NAME, value="cyclic").click() send_post(form, self.options.name, "noko", @@ -126,5 +127,5 @@ class TestPosting(SeleniumTestCase): threadID = threadRE.findall(cur_url)[0][1] replies = self.driver.find_elements(by=By.CSS_SELECTOR, value="div.reply") self.assertEqual(len(replies), self.options.cyclic_count, "Verify that the cyclic thread has the correct number of replies") - self.assertEqual(replies[0].find_element(by=By.CSS_SELECTOR, value="div.post-text").text, "3", "Verify that the first reply is the third post") + self.assertEqual(replies[0].find_element(by=By.CSS_SELECTOR, value="div.post-text").text, "Reply 3", "Verify that the first reply is the third post") delete_post(self.options, int(threadID), self.options.post_password) From 07cf2e58bc12187320abc4f5ee18a1a20b2602a1 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 19 Jan 2025 12:18:05 -0800 Subject: [PATCH 076/122] Add clarification to failing cyclic test message --- tools/selenium_testing/tests/test_posting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/selenium_testing/tests/test_posting.py b/tools/selenium_testing/tests/test_posting.py index 12d6264d..73aad076 100644 --- a/tools/selenium_testing/tests/test_posting.py +++ b/tools/selenium_testing/tests/test_posting.py @@ -126,6 +126,6 @@ class TestPosting(SeleniumTestCase): cur_url = self.driver.current_url threadID = threadRE.findall(cur_url)[0][1] replies = self.driver.find_elements(by=By.CSS_SELECTOR, value="div.reply") - self.assertEqual(len(replies), self.options.cyclic_count, "Verify that the cyclic thread has the correct number of replies") + self.assertEqual(len(replies), self.options.cyclic_count, f"Verify that the cyclic thread has the correct number of replies (CyclicThreadNumPosts in /{self.options.cyclic_board}/board.json must be set to {self.options.cyclic_count})") self.assertEqual(replies[0].find_element(by=By.CSS_SELECTOR, value="div.post-text").text, "Reply 3", "Verify that the first reply is the third post") delete_post(self.options, int(threadID), self.options.post_password) From 1463bbaa63c6b790ee0978791a3db8c82f2ddf0d Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 19 Jan 2025 12:27:24 -0800 Subject: [PATCH 077/122] Update post handler, use EnableCyclicThreads instead of CyclicThreadNumPosts for validation --- pkg/posting/post.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/posting/post.go b/pkg/posting/post.go index 714019d0..8bb0a483 100644 --- a/pkg/posting/post.go +++ b/pkg/posting/post.go @@ -334,7 +334,7 @@ func MakePost(writer http.ResponseWriter, request *http.Request) { } isCyclic := request.PostFormValue("cyclic") == "on" - if isCyclic && boardConfig.CyclicThreadNumPosts == 0 { + if isCyclic && !boardConfig.EnableCyclicThreads { writer.WriteHeader(http.StatusBadRequest) server.ServeError(writer, "Board does not support cyclic threads", wantsJSON, nil) return From e2b5883e514f1f014b309aae84fa0b7a16151952 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 19 Jan 2025 15:04:03 -0800 Subject: [PATCH 078/122] Start implementing in-place migration for pre2021 --- cmd/gochan-migration/internal/pre2021/boards.go | 6 ++++-- cmd/gochan-migration/internal/pre2021/boards_test.go | 2 +- cmd/gochan-migration/internal/pre2021/pre2021_test.go | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index b16b2981..3f203021 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -22,8 +22,10 @@ type migrationSection struct { } func (m *Pre2021Migrator) migrateSectionsInPlace() error { - err := common.NewMigrationError("pre2021", "migrateSectionsInPlace not implemented") - common.LogError().Err(err).Caller().Msg("Failed to migrate sections") + _, err := m.db.ExecSQL(`ALTER TABLE DBPREFIXsections RENAME COLUMN list_order TO position`) + if err != nil { + common.LogError().Caller().Msg("Failed to rename list_order column to position") + } return err } diff --git a/cmd/gochan-migration/internal/pre2021/boards_test.go b/cmd/gochan-migration/internal/pre2021/boards_test.go index 2f06e282..dcb78e1d 100644 --- a/cmd/gochan-migration/internal/pre2021/boards_test.go +++ b/cmd/gochan-migration/internal/pre2021/boards_test.go @@ -73,7 +73,7 @@ func TestMigrateBoardsInPlace(t *testing.T) { t.FailNow() } - if !assert.Error(t, migrator.MigrateBoards(), "Not yet implemented") { + if !assert.NoError(t, migrator.MigrateBoards()) { t.FailNow() } } diff --git a/cmd/gochan-migration/internal/pre2021/pre2021_test.go b/cmd/gochan-migration/internal/pre2021/pre2021_test.go index 02e23733..3ab74379 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021_test.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021_test.go @@ -50,8 +50,8 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 } assert.NoError(t, oldDbFile.Close()) assert.NoError(t, newDbFile.Close()) - migratedDBHost = dbHost - migratedDBName = dbName + dbHost = migratedDBHost + dbName = migratedDBName } oldSQLConfig := config.SQLConfig{ From baa4a10389e3fd3e8131fd6b612c3b80f023fad4 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 19 Jan 2025 21:16:16 -0800 Subject: [PATCH 079/122] Add option to increase line height --- LICENSE | 2 +- frontend/sass/global.scss | 36 ++++++++++++++++++++---------------- frontend/ts/settings.ts | 13 ++++++++++++- html/css/global.css | 4 ++++ 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/LICENSE b/LICENSE index 007de94e..d05f0774 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013-2024, Gochan development group +Copyright (c) 2013-2025, Gochan development group All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/frontend/sass/global.scss b/frontend/sass/global.scss index 1fcd8c6e..339ebd48 100644 --- a/frontend/sass/global.scss +++ b/frontend/sass/global.scss @@ -6,20 +6,24 @@ @import "global/watcher"; @import 'global/bans'; +.increase-line-height { + line-height: 1.5; +} + header { - margin-top:50px; - text-align:center; + margin-top: 50px; + text-align: center; h1 { margin: 0px; } } div#topbar { - left:0; - position:fixed; - top:0; - width:100%; - margin-top:0px; + left: 0; + position: fixed; + top: 0; + width: 100%; + margin-top: 0px; * { padding: 4px; @@ -63,15 +67,15 @@ div#staffmenu { div.section-block { margin-bottom: 8px; div.section-title-block { - display:block; - padding:4px 8px 4px 8px; + display: block; + padding: 4px 8px 4px 8px; } div.section-body { overflow-y: hidden; margin-right: 0px; margin-bottom:8px; min-height: 48px; - padding:8px; + padding: 8px; } } @@ -81,12 +85,12 @@ div.section-block { } #footer { - bottom:0px; - clear:both; - left:0px; - position:static; - text-align:center; - width:100%; + bottom: 0px; + clear: both; + left: 0px; + position: static; + text-align: center; + width: 100%; } select.post-actions { diff --git a/frontend/ts/settings.ts b/frontend/ts/settings.ts index 190f8927..26ef4961 100755 --- a/frontend/ts/settings.ts +++ b/frontend/ts/settings.ts @@ -107,6 +107,7 @@ interface MinMax { min?: number; max?: number; } + class NumberSetting extends Setting { constructor(key: string, title: string, defaultVal = 0, minMax: MinMax = {min: null, max: null}, onSave?:()=>any) { super(key, title, defaultVal, onSave); @@ -205,6 +206,15 @@ export function setTheme() { else themeElem.setAttribute("href", path.join(webroot ?? "/", "css", style)); } + setLineHeight(); +} + +function setLineHeight() { + if(getBooleanStorageVal("increaselineheight", false)) { + document.body.classList.add("increase-line-height"); + } else { + document.body.classList.remove("increase-line-height"); + } } /** @@ -249,16 +259,17 @@ $(() => { } }) as Setting); settings.set("pintopbar", new BooleanSetting("pintopbar", "Pin top bar", true, initTopBar)); + settings.set("increaselineheight", new BooleanSetting("increaselineheight", "Increase line height", false, setLineHeight)); settings.set("enableposthover", new BooleanSetting("enableposthover", "Preview post on hover", true, initPostPreviews)); settings.set("enablepostclick", new BooleanSetting("enablepostclick", "Preview post on click", true, initPostPreviews)); settings.set("useqr", new BooleanSetting("useqr", "Use Quick Reply box", true, () => { if(getBooleanStorageVal("useqr", true)) initQR(); else closeQR(); })); + settings.set("persistentqr", new BooleanSetting("persistentqr", "Persistent Quick Reply", false)); settings.set("watcherseconds", new NumberSetting("watcherseconds", "Check watched threads every # seconds", 15, { min: 2 }, initWatcher)); - settings.set("persistentqr", new BooleanSetting("persistentqr", "Persistent Quick Reply", false)); settings.set("newuploader", new BooleanSetting("newuploader", "Use new upload element", true, updateBrowseButton)); settings.set("customjs", new TextSetting("customjs", "Custom JavaScript", "")); diff --git a/html/css/global.css b/html/css/global.css index ada31b9b..e2da2500 100644 --- a/html/css/global.css +++ b/html/css/global.css @@ -597,6 +597,10 @@ img#banpage-image { margin: 4px 8px 8px 4px; } +.increase-line-height { + line-height: 1.5; +} + header { margin-top: 50px; text-align: center; From 3dc6f3494e2161894a4b7c598fa0748d62f8d336 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 19 Jan 2025 22:40:13 -0800 Subject: [PATCH 080/122] Adjust theme colors --- examples/configs/gochan.example.json | 3 +++ frontend/sass/_util.scss | 12 +++++++++ frontend/sass/_yotsubacommon.scss | 4 ++- frontend/sass/burichan.scss | 26 ++++++++++++------- frontend/sass/burichan/_img.scss | 16 +++++++----- frontend/sass/dark.scss | 5 ---- html/css/burichan.css | 39 +++++++++++++++++++++------- html/css/yotsuba.css | 10 +++++++ html/css/yotsubab.css | 10 +++++++ 9 files changed, 92 insertions(+), 33 deletions(-) diff --git a/examples/configs/gochan.example.json b/examples/configs/gochan.example.json index b07c1972..9107fd21 100644 --- a/examples/configs/gochan.example.json +++ b/examples/configs/gochan.example.json @@ -91,6 +91,9 @@ "somemod:blue" ], "BanMessage": "USER WAS BANNED FOR THIS POST", + + "EnableCyclicThreads": true, + "CyclicThreadNumPosts": 500, "EnableEmbeds": true, "EnableNoFlag": true, "EmbedWidth": 200, diff --git a/frontend/sass/_util.scss b/frontend/sass/_util.scss index 3cff3e41..f42e5333 100644 --- a/frontend/sass/_util.scss +++ b/frontend/sass/_util.scss @@ -18,4 +18,16 @@ -ms-border-radius: $properties; -webkit-border-radius: $properties; border-radius: $properties; +} + +@mixin upload-box($bg, $a, $a-visited) { + div#upload-box { + background: $bg; + a { + color: $a; + } + a:hover, a:target, a:focus { + color: $a-visited; + } + } } \ No newline at end of file diff --git a/frontend/sass/_yotsubacommon.scss b/frontend/sass/_yotsubacommon.scss index a959ce59..3fee917b 100644 --- a/frontend/sass/_yotsubacommon.scss +++ b/frontend/sass/_yotsubacommon.scss @@ -1,4 +1,4 @@ - +@import 'util'; @mixin yotsuba( $fadepath, @@ -101,4 +101,6 @@ color: $subjectcol; font-weight: 700; } + + @include upload-box(#aaa, #444, #666); } \ No newline at end of file diff --git a/frontend/sass/burichan.scss b/frontend/sass/burichan.scss index c2e504a2..a1b6d7af 100644 --- a/frontend/sass/burichan.scss +++ b/frontend/sass/burichan.scss @@ -36,8 +36,7 @@ h3 { font-size: medium; } -h1,h2 { - // background:#D6DAF0; +h1, h2 { font-family: $hfont-family; } @@ -47,21 +46,28 @@ h1, h2, h3 { } div#topbar { - // @include box-shadow(3px 3px 5px 6px $shadowcol); @include shadow-filter(3px 5px 6px $shadowcol); - height: 30px; + min-height: 1.5em; } div#topbar, -.topbar-item, -.topbar-item:visited, +a.topbar-item, +a.topbar-item:visited, .dropdown-button, -.dropdown-menu/* , -.dropdown-menu a */{ - background: #000A89; +div.dropdown-menu { + background: #080e5e; color: $bgcol; } +div.dropdown-menu a, +div.dropdown-menu a:visited { + color: $bgcol; +} + +a.topbar-item:hover, a.topbar-item:active { + background: #0a127b; +} + div#footer { - font-size: 8pt; + font-size: 0.75em; } \ No newline at end of file diff --git a/frontend/sass/burichan/_img.scss b/frontend/sass/burichan/_img.scss index be894261..1d9af926 100644 --- a/frontend/sass/burichan/_img.scss +++ b/frontend/sass/burichan/_img.scss @@ -3,7 +3,7 @@ h1#board-title { font-family: serif; - font-size: 24pt; + font-size: 2em; color: $headercol; } @@ -12,15 +12,15 @@ h1#board-title { } div.file-info { - font-size:12px; - font-family:sans-serif; + font-size: 1em; + font-family: sans-serif; } span.postername { - font-size:12px; - font-family:serif; + font-size: 1em; + font-family: serif; color: $namecol; - font-weight:800; + font-weight: 800; } div.reply, @@ -34,4 +34,6 @@ select.post-actions, div.postprev, div.inlinepostprev { border: 1px solid $bordercol; -} \ No newline at end of file +} + +@include upload-box(#aaa, #444, #666); \ No newline at end of file diff --git a/frontend/sass/dark.scss b/frontend/sass/dark.scss index 00bc411c..8c0185ce 100644 --- a/frontend/sass/dark.scss +++ b/frontend/sass/dark.scss @@ -25,7 +25,6 @@ header { } div#content { - // input:not([type=submit]):not([type=button]), input:not(div#qrbuttons input), textarea, select { background: $inputbg; @@ -41,10 +40,6 @@ th.postblock, table.mgmt-table tr:first-of-type th { text-align: left; } -// select.post-actions { -// color: $color; -// } - div.reply, div.postprev, div.inlinepostprev { diff --git a/html/css/burichan.css b/html/css/burichan.css index 47f20708..a22e1235 100644 --- a/html/css/burichan.css +++ b/html/css/burichan.css @@ -1,6 +1,6 @@ h1#board-title { font-family: serif; - font-size: 24pt; + font-size: 2em; color: #AF0A0F; } @@ -9,12 +9,12 @@ h1#board-title { } div.file-info { - font-size: 12px; + font-size: 1em; font-family: sans-serif; } span.postername { - font-size: 12px; + font-size: 1em; font-family: serif; color: #117743; font-weight: 800; @@ -33,6 +33,16 @@ div.inlinepostprev { border: 1px solid #9295a4; } +div#upload-box { + background: #aaa; +} +div#upload-box a { + color: #444; +} +div#upload-box a:hover, div#upload-box a:target, div#upload-box a:focus { + color: #666; +} + #site-title { font-family: sans-serif; padding-top: 88px; @@ -114,18 +124,27 @@ h1, h2, h3 { div#topbar { -webkit-filter: drop-shadow(3px 5px 6px #555555); filter: drop-shadow(3px 5px 6px #555555); - height: 30px; + min-height: 1.5em; } div#topbar, -.topbar-item, -.topbar-item:visited, +a.topbar-item, +a.topbar-item:visited, .dropdown-button, -.dropdown-menu { - background: #000A89; +div.dropdown-menu { + background: #080e5e; color: #EEF2FF; } -div#footer { - font-size: 8pt; +div.dropdown-menu a, +div.dropdown-menu a:visited { + color: #EEF2FF; +} + +a.topbar-item:hover, a.topbar-item:active { + background: #0a127b; +} + +div#footer { + font-size: 0.75em; } diff --git a/html/css/yotsuba.css b/html/css/yotsuba.css index 193bd3c4..37cb8bf8 100644 --- a/html/css/yotsuba.css +++ b/html/css/yotsuba.css @@ -93,3 +93,13 @@ span.subject { color: #CC1105; font-weight: 700; } + +div#upload-box { + background: #aaa; +} +div#upload-box a { + color: #444; +} +div#upload-box a:hover, div#upload-box a:target, div#upload-box a:focus { + color: #666; +} diff --git a/html/css/yotsubab.css b/html/css/yotsubab.css index b8952f4f..80793e16 100644 --- a/html/css/yotsubab.css +++ b/html/css/yotsubab.css @@ -93,3 +93,13 @@ span.subject { color: #0F0C5D; font-weight: 700; } + +div#upload-box { + background: #aaa; +} +div#upload-box a { + color: #444; +} +div#upload-box a:hover, div#upload-box a:target, div#upload-box a:focus { + color: #666; +} From d48cec2f2be38d62fb22bce93577d086dad5a4e0 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 19 Jan 2025 23:49:01 -0800 Subject: [PATCH 081/122] More CSS style adjustments --- frontend/sass/bunkerchan.scss | 12 +++++++++++- frontend/sass/burichan.scss | 13 ++++++++----- frontend/sass/burichan/_img.scss | 1 + frontend/sass/clear.scss | 6 ++++-- frontend/sass/dark.scss | 9 ++++++++- frontend/sass/photon.scss | 4 +++- html/css/bunkerchan.css | 12 +++++++++++- html/css/burichan.css | 13 +++++++++---- html/css/clear.css | 12 +++++++++++- html/css/dark.css | 8 +++++++- html/css/photon.css | 10 ++++++++++ 11 files changed, 83 insertions(+), 17 deletions(-) diff --git a/frontend/sass/bunkerchan.scss b/frontend/sass/bunkerchan.scss index c26b5447..73dde2d3 100644 --- a/frontend/sass/bunkerchan.scss +++ b/frontend/sass/bunkerchan.scss @@ -30,11 +30,21 @@ div#staff, select.post-actions { div#topbar, div#topbar a, -div#topbar a:visited { +div#topbar a:visited, +div.dropdown-menu { background: $topbarbg; border-bottom: 1px solid $topborder; } +div.dropdown-menu { + border-left: 1px solid $topborder; + border-right: 1px solid $topborder; +} + +div.dropdown-menu a:hover, div.dropdown-menu a:active { + background: $bgcol; +} + table#pages, table#pages * { border: none; } \ No newline at end of file diff --git a/frontend/sass/burichan.scss b/frontend/sass/burichan.scss index a1b6d7af..871c8774 100644 --- a/frontend/sass/burichan.scss +++ b/frontend/sass/burichan.scss @@ -32,7 +32,6 @@ h2 a { h3 { margin: 0px; - text-align: center; font-size: medium; } @@ -59,12 +58,16 @@ div.dropdown-menu { color: $bgcol; } -div.dropdown-menu a, -div.dropdown-menu a:visited { - color: $bgcol; +div.dropdown-menu { + @include shadow-filter(3px 5px 6px $shadowcol); + a, h3 { + color: $bgcol; + } } -a.topbar-item:hover, a.topbar-item:active { +a.topbar-item:hover, +a.topbar-item:active, +div.dropdown-menu a:hover { background: #0a127b; } diff --git a/frontend/sass/burichan/_img.scss b/frontend/sass/burichan/_img.scss index 1d9af926..32a20350 100644 --- a/frontend/sass/burichan/_img.scss +++ b/frontend/sass/burichan/_img.scss @@ -31,6 +31,7 @@ div.inlinepostprev { } select.post-actions, +div.reply, div.postprev, div.inlinepostprev { border: 1px solid $bordercol; diff --git a/frontend/sass/clear.scss b/frontend/sass/clear.scss index 84fb9d5d..356b173f 100644 --- a/frontend/sass/clear.scss +++ b/frontend/sass/clear.scss @@ -64,7 +64,7 @@ table.mgmt-table tr:first-of-type th { } span.subject { - font-weight: bold; + font-weight: 800; color: $subjectcol; } @@ -75,4 +75,6 @@ table#pages, table#pages * { div.section-title-block { background: #A7A7A7; border-bottom: 1px solid $replyborder; -} \ No newline at end of file +} + +@include upload-box(#aaa, #444, #666); \ No newline at end of file diff --git a/frontend/sass/dark.scss b/frontend/sass/dark.scss index 8c0185ce..c28c2078 100644 --- a/frontend/sass/dark.scss +++ b/frontend/sass/dark.scss @@ -4,7 +4,7 @@ body { background: $bgcol; font-family: $font-family; color: $color; - font-size: 80%; + // font-size: 80%; } a { @@ -60,6 +60,13 @@ span.tripcode { color: $headercol; } +.dropdown-menu { + background: $topbarbg!important; + b { + color: black; + } +} + span.subject { color: $subjectcol; } diff --git a/frontend/sass/photon.scss b/frontend/sass/photon.scss index c4dfa7d1..18a16682 100644 --- a/frontend/sass/photon.scss +++ b/frontend/sass/photon.scss @@ -37,4 +37,6 @@ header, div#top-pane, a { #site-title, #board-title { font-weight: 800; -} \ No newline at end of file +} + +@include upload-box(#aaa, #444, #666); \ No newline at end of file diff --git a/html/css/bunkerchan.css b/html/css/bunkerchan.css index edd4f384..783c27c4 100644 --- a/html/css/bunkerchan.css +++ b/html/css/bunkerchan.css @@ -83,11 +83,21 @@ div#staff, select.post-actions { div#topbar, div#topbar a, -div#topbar a:visited { +div#topbar a:visited, +div.dropdown-menu { background: #151515; border-bottom: 1px solid #B0790A; } +div.dropdown-menu { + border-left: 1px solid #B0790A; + border-right: 1px solid #B0790A; +} + +div.dropdown-menu a:hover, div.dropdown-menu a:active { + background: #1D1F21; +} + table#pages, table#pages * { border: none; } diff --git a/html/css/burichan.css b/html/css/burichan.css index a22e1235..66ab6993 100644 --- a/html/css/burichan.css +++ b/html/css/burichan.css @@ -28,6 +28,7 @@ div.inlinepostprev { } select.post-actions, +div.reply, div.postprev, div.inlinepostprev { border: 1px solid #9295a4; @@ -108,7 +109,6 @@ h2 a { h3 { margin: 0px; - text-align: center; font-size: medium; } @@ -136,12 +136,17 @@ div.dropdown-menu { color: #EEF2FF; } -div.dropdown-menu a, -div.dropdown-menu a:visited { +div.dropdown-menu { + -webkit-filter: drop-shadow(3px 5px 6px #555555); + filter: drop-shadow(3px 5px 6px #555555); +} +div.dropdown-menu a, div.dropdown-menu h3 { color: #EEF2FF; } -a.topbar-item:hover, a.topbar-item:active { +a.topbar-item:hover, +a.topbar-item:active, +div.dropdown-menu a:hover { background: #0a127b; } diff --git a/html/css/clear.css b/html/css/clear.css index a6e38551..a86e763a 100644 --- a/html/css/clear.css +++ b/html/css/clear.css @@ -59,7 +59,7 @@ table.mgmt-table tr:first-of-type th { } span.subject { - font-weight: bold; + font-weight: 800; color: #34ED3A; } @@ -71,3 +71,13 @@ div.section-title-block { background: #A7A7A7; border-bottom: 1px solid #117743; } + +div#upload-box { + background: #aaa; +} +div#upload-box a { + color: #444; +} +div#upload-box a:hover, div#upload-box a:target, div#upload-box a:focus { + color: #666; +} diff --git a/html/css/dark.css b/html/css/dark.css index 3cb99307..c634ba95 100644 --- a/html/css/dark.css +++ b/html/css/dark.css @@ -2,7 +2,6 @@ body { background: #1E1E1E; font-family: sans-serif; color: #999; - font-size: 80%; } a { @@ -56,6 +55,13 @@ span.tripcode { color: #32DD72; } +.dropdown-menu { + background: #666 !important; +} +.dropdown-menu b { + color: black; +} + span.subject { color: #446655; } diff --git a/html/css/photon.css b/html/css/photon.css index edc67205..d727af11 100644 --- a/html/css/photon.css +++ b/html/css/photon.css @@ -65,3 +65,13 @@ header, div#top-pane, a { #site-title, #board-title { font-weight: 800; } + +div#upload-box { + background: #aaa; +} +div#upload-box a { + color: #444; +} +div#upload-box a:hover, div#upload-box a:target, div#upload-box a:focus { + color: #666; +} From 65d4dee6734b7a7281ff2c73b00dfb710ddaf287 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sat, 25 Jan 2025 11:41:26 -0800 Subject: [PATCH 082/122] Add alter statements for in-place board migration --- .../internal/pre2021/boards.go | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index 3f203021..6398c8c0 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -10,6 +10,28 @@ import ( "github.com/rs/zerolog" ) +var ( + alterStatements = []string{ + "ALTER TABLE DBPREFIXboards RENAME COLUMN section TO section_id", + "ALTER TABLE DBPREFIXboards RENAME COLUMN list_order TO navbar_position", + "ALTER TABLE DBPREFIXboards RENAME COLUMN created_on TO created_at", + "ALTER TABLE DBPREFIXboards RENAME COLUMN anonymous TO anonymous_name", + "ALTER TABLE DBPREFIXboards RENAME COLUMN forced_anon TO force_anonymous", + "ALTER TABLE DBPREFIXboards RENAME COLUMN embeds_allowed TO allow_embeds", + "ALTER TABLE DBPREFIXboards ADD COLUMN uri VARCHAR(45) NOT NULL", + "ALTER TABLE DBPREFIXboards ADD COLUMN min_message_length SMALLINT NOT NULL", + "ALTER TABLE DBPREFIXboards ADD COLUMN max_threads SMALLINT NOT NULL", + "ALTER TABLE DBPREFIXboards DROP COLUMN type", + "ALTER TABLE DBPREFIXboards DROP COLUMN upload_type", + "ALTER TABLE DBPREFIXboards DROP COLUMN max_age", + // the following statements don't work in SQLite since it doesn't support adding foreign keys after table creation. + // "in-place" migration support for SQLite may be removed + "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_section_id_fk FOREIGN KEY (section_id) REFERENCES DBPREFIXsections(id)", + "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_dir_unique UNIQUE (dir)", + "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_uri_unique UNIQUE (uri)", + } +) + type migrationBoard struct { oldSectionID int oldID int From 12350cc33f631392d9d7b0710d497ca38ad346be Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sat, 25 Jan 2025 13:01:52 -0800 Subject: [PATCH 083/122] Add warning when migrating a SQLite database in-place --- cmd/gochan-migration/main.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/gochan-migration/main.go b/cmd/gochan-migration/main.go index e9941f73..78c287e1 100644 --- a/cmd/gochan-migration/main.go +++ b/cmd/gochan-migration/main.go @@ -95,8 +95,14 @@ func main() { Msg("Starting database migration") config.InitConfig(versionStr) + sqlCfg := config.GetSQLConfig() + if migratingInPlace && sqlCfg.DBtype == "sqlite3" && !updateDB { + common.LogWarning(). + Str("dbType", sqlCfg.DBtype). + Bool("migrateInPlace", migratingInPlace). + Msg("SQLite has limitations with table column changes") + } if !migratingInPlace { - sqlCfg := config.GetSQLConfig() err = gcsql.ConnectToDB(&sqlCfg) if err != nil { fatalEv.Err(err).Caller().Msg("Failed to connect to the database") From cec68cb29beb961a95344b8efd75bbb6ffe382a8 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sat, 25 Jan 2025 13:34:33 -0800 Subject: [PATCH 084/122] Implement in-place board migration --- .../internal/pre2021/boards.go | 41 ++++++++----------- .../internal/pre2021/boards_test.go | 11 +++++ .../internal/pre2021/queries.go | 19 +++++++++ 3 files changed, 46 insertions(+), 25 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index 6398c8c0..5c4ef912 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -10,28 +10,6 @@ import ( "github.com/rs/zerolog" ) -var ( - alterStatements = []string{ - "ALTER TABLE DBPREFIXboards RENAME COLUMN section TO section_id", - "ALTER TABLE DBPREFIXboards RENAME COLUMN list_order TO navbar_position", - "ALTER TABLE DBPREFIXboards RENAME COLUMN created_on TO created_at", - "ALTER TABLE DBPREFIXboards RENAME COLUMN anonymous TO anonymous_name", - "ALTER TABLE DBPREFIXboards RENAME COLUMN forced_anon TO force_anonymous", - "ALTER TABLE DBPREFIXboards RENAME COLUMN embeds_allowed TO allow_embeds", - "ALTER TABLE DBPREFIXboards ADD COLUMN uri VARCHAR(45) NOT NULL", - "ALTER TABLE DBPREFIXboards ADD COLUMN min_message_length SMALLINT NOT NULL", - "ALTER TABLE DBPREFIXboards ADD COLUMN max_threads SMALLINT NOT NULL", - "ALTER TABLE DBPREFIXboards DROP COLUMN type", - "ALTER TABLE DBPREFIXboards DROP COLUMN upload_type", - "ALTER TABLE DBPREFIXboards DROP COLUMN max_age", - // the following statements don't work in SQLite since it doesn't support adding foreign keys after table creation. - // "in-place" migration support for SQLite may be removed - "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_section_id_fk FOREIGN KEY (section_id) REFERENCES DBPREFIXsections(id)", - "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_dir_unique UNIQUE (dir)", - "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_uri_unique UNIQUE (uri)", - } -) - type migrationBoard struct { oldSectionID int oldID int @@ -59,9 +37,22 @@ func (m *Pre2021Migrator) migrateBoardsInPlace() error { errEv.Err(err).Caller().Msg("Failed to migrate sections") return err } - err = common.NewMigrationError("pre2021", "migrateBoardsInPlace not implemented") - errEv.Err(err).Caller().Msg("Failed to migrate boards") - return err + + for _, statement := range alterStatements { + if strings.Contains(statement, "CONSTRAINT") && m.db.SQLDriver() == "sqlite3" { + // skip constraints in SQLite since they can't be added after table creation + continue + } + _, err = m.db.ExecSQL(statement) + if err != nil { + errEv.Err(err).Caller(). + Str("statement", statement). + Msg("Failed to execute alter statement") + return err + } + } + + return nil } func (m *Pre2021Migrator) migrateSectionsToNewDB() error { diff --git a/cmd/gochan-migration/internal/pre2021/boards_test.go b/cmd/gochan-migration/internal/pre2021/boards_test.go index dcb78e1d..13909f74 100644 --- a/cmd/gochan-migration/internal/pre2021/boards_test.go +++ b/cmd/gochan-migration/internal/pre2021/boards_test.go @@ -1,7 +1,9 @@ package pre2021 import ( + "context" "testing" + "time" "github.com/gochan-org/gochan/pkg/gcsql" "github.com/stretchr/testify/assert" @@ -76,4 +78,13 @@ func TestMigrateBoardsInPlace(t *testing.T) { if !assert.NoError(t, migrator.MigrateBoards()) { t.FailNow() } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(migrator.config.DBTimeoutSeconds)) + defer cancel() + var uri string + var sectionID int + assert.NoError(t, migrator.db.QueryRowContextSQL(ctx, nil, "SELECT uri FROM DBPREFIXboards WHERE dir = ?", []any{"test"}, []any{&uri})) + assert.NoError(t, migrator.db.QueryRowContextSQL(ctx, nil, "SELECT section_id FROM DBPREFIXboards WHERE dir = ?", []any{"test"}, []any{§ionID})) + assert.Equal(t, "", uri) + assert.Greater(t, sectionID, 0) } diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index b8fbbc68..bb12f15b 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -21,3 +21,22 @@ timestamp, expires, permaban, reason, type, staff_note, appeal_at, can_appeal FR announcementsQuery = "SELECT id, subject, message, poster, timestamp FROM DBPREFIXannouncements" ) + +var ( + alterStatements = []string{ + "ALTER TABLE DBPREFIXboards RENAME COLUMN section TO section_id", + "ALTER TABLE DBPREFIXboards RENAME COLUMN list_order TO navbar_position", + "ALTER TABLE DBPREFIXboards RENAME COLUMN created_on TO created_at", + "ALTER TABLE DBPREFIXboards RENAME COLUMN anonymous TO anonymous_name", + "ALTER TABLE DBPREFIXboards RENAME COLUMN forced_anon TO force_anonymous", + "ALTER TABLE DBPREFIXboards RENAME COLUMN embeds_allowed TO allow_embeds", + "ALTER TABLE DBPREFIXboards ADD COLUMN uri VARCHAR(45) NOT NULL DEFAULT ''", + "ALTER TABLE DBPREFIXboards ADD COLUMN min_message_length SMALLINT NOT NULL DEFAULT 0", + "ALTER TABLE DBPREFIXboards ADD COLUMN max_threads SMALLINT NOT NULL DEFAULT 65535", + // the following statements don't work in SQLite since it doesn't support adding foreign keys after table creation. + // "in-place" migration support for SQLite may be removed + "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_section_id_fk FOREIGN KEY (section_id) REFERENCES DBPREFIXsections(id)", + "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_dir_unique UNIQUE (dir)", + "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_uri_unique UNIQUE (uri)", + } +) From f0f0f055da1fda420d47cdb81afcbf3da292a105 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sat, 25 Jan 2025 17:46:55 -0800 Subject: [PATCH 085/122] Add post migration in place --- .../internal/pre2021/boards.go | 2 +- .../internal/pre2021/posts.go | 152 +++++++++++++++++- .../internal/pre2021/posts_test.go | 29 ++++ .../internal/pre2021/pre2021_test.go | 15 +- .../internal/pre2021/queries.go | 15 +- .../internal/pre2021/staff.go | 4 +- 6 files changed, 210 insertions(+), 7 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index 5c4ef912..ce1f2e83 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -38,7 +38,7 @@ func (m *Pre2021Migrator) migrateBoardsInPlace() error { return err } - for _, statement := range alterStatements { + for _, statement := range boardAlterStatements { if strings.Contains(statement, "CONSTRAINT") && m.db.SQLDriver() == "sqlite3" { // skip constraints in SQLite since they can't be added after table creation continue diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index 1f786a3b..ea8b8f96 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -3,6 +3,8 @@ package pre2021 import ( "context" "database/sql" + "os" + "strings" "time" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" @@ -195,5 +197,153 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { } func (m *Pre2021Migrator) migratePostsInPlace() error { - return common.NewMigrationError("pre2021", "not yet implemented") + errEv := common.LogError() + defer errEv.Discard() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(m.config.DBTimeoutSeconds)) + defer cancel() + + ba, err := os.ReadFile(gcutil.FindResource("sql/initdb_" + m.db.SQLDriver() + ".sql")) + if err != nil { + errEv.Err(err).Caller(). + Msg("Failed to read initdb SQL file") + return err + } + statements := strings.Split(string(ba), ";") + for _, statement := range statements { + statement = strings.TrimSpace(statement) + if strings.HasPrefix(statement, "CREATE TABLE DBPREFIXthreads") || strings.HasPrefix(statement, "CREATE TABLE DBPREFIXfiles") { + if _, err = m.db.ExecContextSQL(ctx, nil, statement); err != nil { + errEv.Err(err).Caller().Msg("Failed to create threads table") + return err + } + } + } + + rows, err := m.db.QueryContextSQL(ctx, nil, threadsQuery+" AND parentid = 0") + if err != nil { + errEv.Err(err).Caller(). + Msg("Failed to get threads") + return err + } + defer rows.Close() + + var threads []migrationPost + for rows.Next() { + var post migrationPost + if err = rows.Scan( + &post.ID, &post.oldBoardID, &post.oldParentID, &post.Name, &post.Tripcode, &post.Email, + &post.Subject, &post.Message, &post.MessageRaw, &post.Password, &post.filename, + &post.filenameOriginal, &post.fileChecksum, &post.filesize, &post.imageW, &post.imageH, + &post.thumbW, &post.thumbH, &post.IP, &post.CreatedOn, &post.autosage, + &post.bumped, &post.stickied, &post.locked, + ); err != nil { + errEv.Err(err).Caller(). + Msg("Failed to scan thread") + return err + } + threads = append(threads, post) + } + if err = rows.Close(); err != nil { + errEv.Caller().Msg("Failed to close thread rows") + return err + } + + for _, statements := range postAlterStatements { + if _, err = m.db.ExecContextSQL(ctx, nil, statements); err != nil { + errEv.Err(err).Caller().Msg("Failed to alter posts table") + return err + } + } + + switch m.db.SQLDriver() { + case "mysql": + _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts ADD COLUMN ip_new VARBINARY(16) NOT NULL") + case "postgres", "postgresql": + _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts ADD COLUMN ip_new INET NOT NULL") + case "sqlite3": + _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts ADD COLUMN ip_new VARCHAR(45) NOT NULL DEFAULT '0.0.0.0'") + } + if err != nil { + errEv.Err(err).Caller().Msg("Failed to update IP column") + return err + } + if _, err = m.db.ExecContextSQL(ctx, nil, "UPDATE DBPREFIXposts SET ip_new = IP_ATON"); err != nil { + errEv.Err(err).Caller().Msg("Failed to update IP column") + return err + } + if _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts RENAME COLUMN ip TO ip_old"); err != nil { + errEv.Err(err).Caller().Msg("Failed to rename old IP column") + return err + } + if _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts RENAME COLUMN ip_new TO ip"); err != nil { + errEv.Err(err).Caller().Msg("Failed to rename new IP column") + return err + } + + for _, op := range threads { + if _, err = m.db.ExecContextSQL(ctx, nil, + `INSERT INTO DBPREFIXthreads(board_id,locked,stickied,anchored,cyclical,last_bump,is_deleted) VALUES(?,?,?,?,?,?,?)`, + op.oldBoardID, op.locked, op.stickied, op.autosage, false, op.bumped, false, + ); err != nil { + errEv.Err(err).Caller(). + Int("postID", op.ID). + Msg("Failed to insert thread") + return err + } + if err = m.db.QueryRowContextSQL(ctx, nil, "SELECT MAX(id) FROM DBPREFIXthreads", nil, []any{&op.ThreadID}); err != nil { + errEv.Err(err).Caller(). + Int("postID", op.ID). + Msg("Failed to get thread ID") + return err + } + + if _, err = m.db.ExecContextSQL(ctx, nil, + "UPDATE DBPREFIXposts SET thread_id = ? WHERE (id = ? and is_top_post) or thread_id = ?", op.ThreadID, op.oldID, op.oldID, + ); err != nil { + errEv.Err(err).Caller(). + Int("postID", op.ID). + Int("threadID", op.ThreadID). + Msg("Failed to set thread ID") + return err + } + } + if rows, err = m.db.QueryContextSQL(ctx, nil, + "SELECT id,filename,filename_original,file_checksum,filesize,image_w,image_h,thumb_w,thumb_h FROM DBPREFIXposts WHERE filename <> ''", + ); err != nil { + errEv.Err(err).Caller().Msg("Failed to get uploads") + return err + } + defer rows.Close() + + var uploads []gcsql.Upload + for rows.Next() { + var upload gcsql.Upload + if err = rows.Scan(&upload.PostID, &upload.Filename, &upload.OriginalFilename, &upload.Checksum, &upload.FileSize, &upload.Width, + &upload.Height, &upload.ThumbnailWidth, &upload.ThumbnailHeight, + ); err != nil { + errEv.Err(err).Caller().Msg("Failed to scan upload") + return err + } + uploads = append(uploads, upload) + } + if err = rows.Close(); err != nil { + errEv.Caller().Msg("Failed to close upload rows") + return err + } + + for _, upload := range uploads { + if _, err = m.db.ExecContextSQL(ctx, nil, + `INSERT INTO DBPREFIXfiles(post_id,file_order,filename,original_filename,checksum,file_size,width,height,thumbnail_width,thumbnail_height,is_spoilered) VALUES + (?,0,?,?,?,?,?,?,?,?,0)`, + upload.PostID, upload.Filename, upload.OriginalFilename, upload.Checksum, upload.FileSize, upload.Width, upload.Height, + upload.ThumbnailWidth, upload.ThumbnailHeight, + ); err != nil { + errEv.Err(err).Caller(). + Int("postID", upload.PostID). + Msg("Failed to insert upload") + return err + } + } + + return nil } diff --git a/cmd/gochan-migration/internal/pre2021/posts_test.go b/cmd/gochan-migration/internal/pre2021/posts_test.go index d55d8d4e..34f38624 100644 --- a/cmd/gochan-migration/internal/pre2021/posts_test.go +++ b/cmd/gochan-migration/internal/pre2021/posts_test.go @@ -49,3 +49,32 @@ func TestMigratePostsToNewDB(t *testing.T) { assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXfiles", nil, []any{&numUploadPosts})) assert.Equal(t, 1, numUploadPosts, "Expected to have 1 upload post") } + +func TestMigratePostsInPlace(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, true) + if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateBoards()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigratePosts()) { + t.FailNow() + } + var numThreads int + if !assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numThreads}), "Failed to get number of threads") { + t.FailNow() + } + assert.Equal(t, 2, numThreads, "Expected to have two threads pre-migration") + + var numUploadPosts int + assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXfiles", nil, []any{&numUploadPosts})) + assert.Equal(t, 1, numUploadPosts, "Expected to have 1 upload post") + + var ip string + assert.NoError(t, gcsql.QueryRowSQL("SELECT IP_NTOA FROM DBPREFIXposts WHERE id = 1", nil, []any{&ip})) + assert.Equal(t, "192.168.56.1", ip, "Expected to have the correct IP address") +} diff --git a/cmd/gochan-migration/internal/pre2021/pre2021_test.go b/cmd/gochan-migration/internal/pre2021/pre2021_test.go index 3ab74379..338b0c31 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021_test.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021_test.go @@ -90,13 +90,24 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 return migrator } -func TestPre2021Migration(t *testing.T) { +func TestPre2021MigrationToNewDB(t *testing.T) { outDir := t.TempDir() migrator := setupMigrationTest(t, outDir, false) if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { t.FailNow() } migrated, err := migrator.MigrateDB() - assert.False(t, migrated) assert.NoError(t, err) + assert.False(t, migrated) +} + +func TestPre2021MigrationInPlace(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, true) + if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { + t.FailNow() + } + migrated, err := migrator.MigrateDB() + assert.NoError(t, err) + assert.False(t, migrated) } diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index bb12f15b..438510ee 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -23,7 +23,7 @@ timestamp, expires, permaban, reason, type, staff_note, appeal_at, can_appeal FR ) var ( - alterStatements = []string{ + boardAlterStatements = []string{ "ALTER TABLE DBPREFIXboards RENAME COLUMN section TO section_id", "ALTER TABLE DBPREFIXboards RENAME COLUMN list_order TO navbar_position", "ALTER TABLE DBPREFIXboards RENAME COLUMN created_on TO created_at", @@ -39,4 +39,17 @@ var ( "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_dir_unique UNIQUE (dir)", "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_uri_unique UNIQUE (uri)", } + postAlterStatements = []string{ + "ALTER TABLE DBPREFIXposts RENAME COLUMN parentid TO thread_id", + "ALTER TABLE DBPREFIXposts RENAME COLUMN timestamp TO created_on", + "ALTER TABLE DBPREFIXposts RENAME COLUMN deleted_timestamp TO deleted_at", + // "ALTER TABLE DBPREFIXposts RENAME COLUMN ip TO ip_old", + "ALTER TABLE DBPREFIXposts ADD COLUMN is_top_post BOOL NOT NULL DEFAULT FALSE", + "ALTER TABLE DBPREFIXposts ADD COLUMN is_role_signature BOOL NOT NULL DEFAULT FALSE", + "ALTER TABLE DBPREFIXposts ADD COLUMN is_deleted BOOL NOT NULL DEFAULT FALSE", + "ALTER TABLE DBPREFIXposts ADD COLUMN banned_message TEXT", + "ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''", + "ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''", + "UPDATE DBPREFIXposts SET is_top_post = TRUE WHERE thread_id = 0", + } ) diff --git a/cmd/gochan-migration/internal/pre2021/staff.go b/cmd/gochan-migration/internal/pre2021/staff.go index d2fb79df..114bacdb 100644 --- a/cmd/gochan-migration/internal/pre2021/staff.go +++ b/cmd/gochan-migration/internal/pre2021/staff.go @@ -12,8 +12,8 @@ import ( ) func (*Pre2021Migrator) migrateStaffInPlace() error { - err := common.NewMigrationError("pre2021", "migrateSectionsInPlace not yet implemented") - common.LogError().Err(err).Caller().Msg("Failed to migrate sections") + err := common.NewMigrationError("pre2021", "migrateStaff not yet implemented") + common.LogError().Err(err).Caller().Msg("Failed to migrate staff") return err } From d3c985bd508c75e3e68a865d166bd50ec7e3d5a0 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 26 Jan 2025 16:24:45 -0800 Subject: [PATCH 086/122] Implement staff and ban migration in place, and the respective tests --- cmd/gochan-migration/internal/pre2021/bans.go | 27 ++++++++++- .../internal/pre2021/bans_test.go | 45 +++++++++++++++++++ .../internal/pre2021/queries.go | 5 +++ .../internal/pre2021/staff.go | 21 +++++++-- .../internal/pre2021/staff_test.go | 17 +++++++ 5 files changed, 110 insertions(+), 5 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/bans.go b/cmd/gochan-migration/internal/pre2021/bans.go index 1fbfdfdb..a7cb25da 100644 --- a/cmd/gochan-migration/internal/pre2021/bans.go +++ b/cmd/gochan-migration/internal/pre2021/bans.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "net" + "os" "strings" "time" @@ -38,7 +39,31 @@ type migrationBan struct { } func (m *Pre2021Migrator) migrateBansInPlace() error { - return common.NewMigrationError("pre2021", "migrateBansInPlace not yet implemented") + errEv := common.LogError() + defer errEv.Discard() + initSQLPath := gcutil.FindResource("sql/initdb_" + m.db.SQLDriver() + ".sql") + ba, err := os.ReadFile(initSQLPath) + if err != nil { + errEv.Err(err).Caller(). + Str("initDBFile", initSQLPath). + Msg("Failed to read initdb file") + return err + } + statements := strings.Split(string(ba), ";") + for _, stmt := range statements { + stmt = strings.TrimSpace(stmt) + if strings.HasPrefix(stmt, "CREATE TABLE DBPREFIXip_ban") || strings.HasPrefix(stmt, "CREATE TABLE DBPREFIXfilter") { + _, err = gcsql.ExecSQL(stmt) + if err != nil { + errEv.Err(err).Caller(). + Str("statement", stmt). + Msg("Failed to create table") + return err + } + } + } + // since the table names are different, migrateBansToNewDB can be called directly to migrate bans + return m.migrateBansToNewDB() } func (m *Pre2021Migrator) migrateBan(tx *sql.Tx, ban *migrationBan, boardID *int, errEv *zerolog.Event) error { diff --git a/cmd/gochan-migration/internal/pre2021/bans_test.go b/cmd/gochan-migration/internal/pre2021/bans_test.go index f527894c..da32d85d 100644 --- a/cmd/gochan-migration/internal/pre2021/bans_test.go +++ b/cmd/gochan-migration/internal/pre2021/bans_test.go @@ -51,3 +51,48 @@ func TestMigrateBansToNewDB(t *testing.T) { } assert.Equal(t, 3, len(conditions), "Expected filter to have three conditions") } + +func TestMigrateBansInPlace(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, true) + if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateBoards()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigratePosts()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateStaff()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateBans()) { + t.FailNow() + } + bans, err := gcsql.GetIPBans(0, 200, false) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, 6, len(bans), "Expected to have 4 valid bans") + assert.NotZero(t, bans[0].StaffID, "Expected ban staff ID field to be set") + + var numInvalidBans int + assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXip_ban WHERE message = ?", []any{"Full ban on 8.8.0.0/16"}, []any{&numInvalidBans})) + assert.Equal(t, 0, numInvalidBans, "Expected the invalid test to not be migrated") + + filters, err := gcsql.GetAllFilters(gcsql.TrueOrFalse) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, 1, len(filters)) + conditions, err := filters[0].Conditions() + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, 3, len(conditions), "Expected filter to have three conditions") +} diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index 438510ee..73a4ac89 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -52,4 +52,9 @@ var ( "ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''", "UPDATE DBPREFIXposts SET is_top_post = TRUE WHERE thread_id = 0", } + staffAlterStatements = []string{ + "ALTER TABLE DBPREFIXstaff RENAME COLUMN rank TO global_rank", + "ALTER TABLE DBPREFIXstaff RENAME COLUMN last_active TO last_login", + "ALTER TABLE DBPREFIXstaff ADD COLUMN is_active BOOL NOT NULL DEFAULT TRUE", + } ) diff --git a/cmd/gochan-migration/internal/pre2021/staff.go b/cmd/gochan-migration/internal/pre2021/staff.go index 114bacdb..cca7d9cd 100644 --- a/cmd/gochan-migration/internal/pre2021/staff.go +++ b/cmd/gochan-migration/internal/pre2021/staff.go @@ -11,10 +11,23 @@ import ( "github.com/rs/zerolog" ) -func (*Pre2021Migrator) migrateStaffInPlace() error { - err := common.NewMigrationError("pre2021", "migrateStaff not yet implemented") - common.LogError().Err(err).Caller().Msg("Failed to migrate staff") - return err +func (m *Pre2021Migrator) migrateStaffInPlace() error { + errEv := common.LogError() + defer errEv.Discard() + + for _, stmt := range staffAlterStatements { + if _, err := gcsql.ExecSQL(stmt); err != nil { + errEv.Err(err).Caller().Msg("Failed to alter staff table") + return err + } + } + + _, err := m.getMigrationUser(errEv) + if err != nil { + return err + } + + return nil } func (m *Pre2021Migrator) getMigrationUser(errEv *zerolog.Event) (*gcsql.Staff, error) { diff --git a/cmd/gochan-migration/internal/pre2021/staff_test.go b/cmd/gochan-migration/internal/pre2021/staff_test.go index 5c81dea3..90c4efbf 100644 --- a/cmd/gochan-migration/internal/pre2021/staff_test.go +++ b/cmd/gochan-migration/internal/pre2021/staff_test.go @@ -27,3 +27,20 @@ func TestMigrateStaffToNewDB(t *testing.T) { } assert.Equal(t, 3, migratedAdmin.Rank) } + +func TestMigrateStaffInPlace(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, true) + if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateStaff()) { + t.FailNow() + } + migratedAdmin, err := gcsql.GetStaffByUsername("migratedadmin", true) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, 3, migratedAdmin.Rank) +} From 5c4e2006312f6236ffa47b089cb4258b033d4441 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 26 Jan 2025 17:01:27 -0800 Subject: [PATCH 087/122] Implement in-place announcement migration --- .../internal/pre2021/announcements.go | 77 ++++++++++++++++++- .../internal/pre2021/queries.go | 2 + 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/announcements.go b/cmd/gochan-migration/internal/pre2021/announcements.go index e885e296..2f8ae52e 100644 --- a/cmd/gochan-migration/internal/pre2021/announcements.go +++ b/cmd/gochan-migration/internal/pre2021/announcements.go @@ -8,12 +8,85 @@ import ( "github.com/gochan-org/gochan/pkg/gcsql" ) -func (*Pre2021Migrator) migrateAnnouncementsInPlace() error { - return common.NewMigrationError("pre2021", "migrateAnnouncementsInPlace not implemented") +func (m *Pre2021Migrator) migrateAnnouncementsInPlace() error { + errEv := common.LogError() + defer errEv.Discard() + + if _, err := gcsql.ExecSQL(announcementsAlterStatement); err != nil { + errEv.Err(err).Caller().Msg("Failed to alter announcements table") + return err + } + + var staffIDs []int + rows, err := m.db.QuerySQL("SELECT id FROM DBPREFIXstaff") + if err != nil { + errEv.Err(err).Caller().Msg("Failed to get staff IDs") + return err + } + defer rows.Close() + for rows.Next() { + var id int + if err = rows.Scan(&id); err != nil { + errEv.Err(err).Caller().Msg("Failed to scan staff ID") + return err + } + staffIDs = append(staffIDs, id) + } + if err = rows.Close(); err != nil { + errEv.Err(err).Caller().Msg("Failed to close staff ID rows") + return err + } + + m.migrationUser, err = m.getMigrationUser(errEv) + if err != nil { + errEv.Err(err).Caller().Msg("Failed to get migration user") + return err + } + + rows, err = m.db.QuerySQL("SELECT poster FROM DBPREFIXannouncements") + if err != nil { + errEv.Err(err).Caller().Msg("Failed to get announcements") + return err + } + defer rows.Close() + + var announcementPosters []string + for rows.Next() { + var poster string + if err = rows.Scan(&poster); err != nil { + errEv.Err(err).Caller().Msg("Failed to scan announcement row") + return err + } + announcementPosters = append(announcementPosters, poster) + } + if err = rows.Close(); err != nil { + errEv.Err(err).Caller().Msg("Failed to close announcement rows") + return err + } + for _, poster := range announcementPosters { + id, err := gcsql.GetStaffID(poster) + if errors.Is(err, gcsql.ErrUnrecognizedUsername) { + // user doesn't exist, use migration user + common.LogWarning().Str("staff", poster).Msg("Staff username not found in database") + id = m.migrationUser.ID + } else if err != nil { + errEv.Err(err).Caller().Str("staff", poster).Msg("Failed to get staff ID") + return err + } + + if _, err = gcsql.ExecSQL("UPDATE DBPREFIXannouncements SET staff_id = ? WHERE poster = ?", id, poster); err != nil { + errEv.Err(err).Caller().Str("staff", poster).Msg("Failed to update announcement poster") + return err + } + } + + return nil } func (m *Pre2021Migrator) migrateAnnouncementsToNewDB() error { errEv := common.LogError() + defer errEv.Discard() + rows, err := m.db.QuerySQL(announcementsQuery) if err != nil { errEv.Err(err).Caller().Msg("Failed to get announcements") diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index 73a4ac89..4342446b 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -20,6 +20,8 @@ bumped, stickied, locked FROM DBPREFIXposts WHERE deleted_timestamp IS NULL` timestamp, expires, permaban, reason, type, staff_note, appeal_at, can_appeal FROM DBPREFIXbanlist` announcementsQuery = "SELECT id, subject, message, poster, timestamp FROM DBPREFIXannouncements" + + announcementsAlterStatement = "ALTER TABLE DBPREFIXannouncements ADD COLUMN staff_id INT NOT NULL DEFAULT 1" ) var ( From 2f39e9d7cceb0b68964fb353518cff2378121c53 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 26 Jan 2025 17:12:23 -0800 Subject: [PATCH 088/122] Add tests for announcement migration --- .../internal/pre2021/announcements.go | 24 +------- .../internal/pre2021/announcements_test.go | 56 ++++++++++++++++++ tools/gochan-pre2021.sqlite3db | Bin 81920 -> 81920 bytes 3 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 cmd/gochan-migration/internal/pre2021/announcements_test.go diff --git a/cmd/gochan-migration/internal/pre2021/announcements.go b/cmd/gochan-migration/internal/pre2021/announcements.go index 2f8ae52e..e015f934 100644 --- a/cmd/gochan-migration/internal/pre2021/announcements.go +++ b/cmd/gochan-migration/internal/pre2021/announcements.go @@ -17,33 +17,14 @@ func (m *Pre2021Migrator) migrateAnnouncementsInPlace() error { return err } - var staffIDs []int - rows, err := m.db.QuerySQL("SELECT id FROM DBPREFIXstaff") - if err != nil { - errEv.Err(err).Caller().Msg("Failed to get staff IDs") - return err - } - defer rows.Close() - for rows.Next() { - var id int - if err = rows.Scan(&id); err != nil { - errEv.Err(err).Caller().Msg("Failed to scan staff ID") - return err - } - staffIDs = append(staffIDs, id) - } - if err = rows.Close(); err != nil { - errEv.Err(err).Caller().Msg("Failed to close staff ID rows") - return err - } - + var err error m.migrationUser, err = m.getMigrationUser(errEv) if err != nil { errEv.Err(err).Caller().Msg("Failed to get migration user") return err } - rows, err = m.db.QuerySQL("SELECT poster FROM DBPREFIXannouncements") + rows, err := m.db.QuerySQL("SELECT poster FROM DBPREFIXannouncements") if err != nil { errEv.Err(err).Caller().Msg("Failed to get announcements") return err @@ -111,6 +92,7 @@ func (m *Pre2021Migrator) migrateAnnouncementsToNewDB() error { // user doesn't exist, use migration user common.LogWarning().Str("staff", staff).Msg("Staff username not found in database") message += "\n(originally by " + staff + ")" + staffID = m.migrationUser.ID } else if err != nil { errEv.Err(err).Caller().Str("staff", staff).Msg("Failed to get staff ID") return err diff --git a/cmd/gochan-migration/internal/pre2021/announcements_test.go b/cmd/gochan-migration/internal/pre2021/announcements_test.go new file mode 100644 index 00000000..b303b40c --- /dev/null +++ b/cmd/gochan-migration/internal/pre2021/announcements_test.go @@ -0,0 +1,56 @@ +package pre2021 + +import ( + "testing" + + "github.com/gochan-org/gochan/pkg/gcsql" + "github.com/stretchr/testify/assert" +) + +func TestMigrateAnnouncementsToNewDB(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, false) + if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateBoards()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateStaff()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateAnnouncements()) { + t.FailNow() + } + + var numAnnouncements int + assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXannouncements WHERE staff_id > 0", nil, []any{&numAnnouncements})) + assert.Equal(t, 2, numAnnouncements, "Expected to have two announcement") +} + +func TestMigrateAnnouncementsInPlace(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, true) + if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateBoards()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateStaff()) { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateAnnouncements()) { + t.FailNow() + } + + var numAnnouncements int + assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXannouncements WHERE staff_id > 0", nil, []any{&numAnnouncements})) + assert.Equal(t, 2, numAnnouncements, "Expected to have two announcement") +} diff --git a/tools/gochan-pre2021.sqlite3db b/tools/gochan-pre2021.sqlite3db index 9c99c2aba1ac978c25bc47d4d5982d02f3015c20..d4dc0035b247919e0cb157f9afa39819ad3c8418 100644 GIT binary patch delta 229 zcmZo@U~On%ogmGqHBrWyRf|Eds&He<0)7rA{tyQK4g4V+8$I}AqL|nivZa-cgG-aL zQj<#*^3xO&^YZdb^O93@Q}ap`$}>wc6iV}oQj_!3^D?VaQxu9z64TOrQ;Ule(^FIO zQ;YLTQY$ixON~r z{FC^b_&ql^>hoKQa5C#MrYFZETUpG+!^A8L5(V4D#Q%svlmQ(yH|cM0(r2s_006B! BLD&EQ delta 113 zcmZo@U~On%ogmGqIZ?)$Rg*!ltb1e10)93|{tXQL8#W6jEaRWNK|V<05d$ZGJOlrG z{&@a#{QLPg@-O0_#NWi9yRp%gUzvxAS(Y(9Ilj0gF)fWrn3GwT6Ua-<%gZm#OHR#A R%_}KxZqnc0q|aC<004}hBIp1B From fcb61cac577d4146cb9b7a92253d5d3bafbe6296 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 26 Jan 2025 22:35:35 -0800 Subject: [PATCH 089/122] Use separate function to validate pre2021 migration results --- .../internal/pre2021/announcements_test.go | 7 ++-- .../internal/pre2021/bans_test.go | 26 +++--------- .../internal/pre2021/boards_test.go | 41 ++++++++----------- .../internal/pre2021/posts_test.go | 38 ++++++++--------- .../internal/pre2021/pre2021_test.go | 10 +++++ .../internal/pre2021/staff_test.go | 10 ++--- 6 files changed, 59 insertions(+), 73 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/announcements_test.go b/cmd/gochan-migration/internal/pre2021/announcements_test.go index b303b40c..6cb4b2da 100644 --- a/cmd/gochan-migration/internal/pre2021/announcements_test.go +++ b/cmd/gochan-migration/internal/pre2021/announcements_test.go @@ -26,9 +26,7 @@ func TestMigrateAnnouncementsToNewDB(t *testing.T) { t.FailNow() } - var numAnnouncements int - assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXannouncements WHERE staff_id > 0", nil, []any{&numAnnouncements})) - assert.Equal(t, 2, numAnnouncements, "Expected to have two announcement") + validateAnnouncementMigration(t) } func TestMigrateAnnouncementsInPlace(t *testing.T) { @@ -49,7 +47,10 @@ func TestMigrateAnnouncementsInPlace(t *testing.T) { if !assert.NoError(t, migrator.MigrateAnnouncements()) { t.FailNow() } + validateAnnouncementMigration(t) +} +func validateAnnouncementMigration(t *testing.T) { var numAnnouncements int assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXannouncements WHERE staff_id > 0", nil, []any{&numAnnouncements})) assert.Equal(t, 2, numAnnouncements, "Expected to have two announcement") diff --git a/cmd/gochan-migration/internal/pre2021/bans_test.go b/cmd/gochan-migration/internal/pre2021/bans_test.go index da32d85d..3efa9412 100644 --- a/cmd/gochan-migration/internal/pre2021/bans_test.go +++ b/cmd/gochan-migration/internal/pre2021/bans_test.go @@ -29,27 +29,8 @@ func TestMigrateBansToNewDB(t *testing.T) { if !assert.NoError(t, migrator.MigrateBans()) { t.FailNow() } - bans, err := gcsql.GetIPBans(0, 200, false) - if !assert.NoError(t, err) { - t.FailNow() - } - assert.Equal(t, 6, len(bans), "Expected to have 4 valid bans") - assert.NotZero(t, bans[0].StaffID, "Expected ban staff ID field to be set") - var numInvalidBans int - assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXip_ban WHERE message = ?", []any{"Full ban on 8.8.0.0/16"}, []any{&numInvalidBans})) - assert.Equal(t, 0, numInvalidBans, "Expected the invalid test to not be migrated") - - filters, err := gcsql.GetAllFilters(gcsql.TrueOrFalse) - if !assert.NoError(t, err) { - t.FailNow() - } - assert.Equal(t, 1, len(filters)) - conditions, err := filters[0].Conditions() - if !assert.NoError(t, err) { - t.FailNow() - } - assert.Equal(t, 3, len(conditions), "Expected filter to have three conditions") + validateBanMigration(t) } func TestMigrateBansInPlace(t *testing.T) { @@ -74,6 +55,11 @@ func TestMigrateBansInPlace(t *testing.T) { if !assert.NoError(t, migrator.MigrateBans()) { t.FailNow() } + + validateBanMigration(t) +} + +func validateBanMigration(t *testing.T) { bans, err := gcsql.GetIPBans(0, 200, false) if !assert.NoError(t, err) { t.FailNow() diff --git a/cmd/gochan-migration/internal/pre2021/boards_test.go b/cmd/gochan-migration/internal/pre2021/boards_test.go index 13909f74..d42152b1 100644 --- a/cmd/gochan-migration/internal/pre2021/boards_test.go +++ b/cmd/gochan-migration/internal/pre2021/boards_test.go @@ -1,9 +1,7 @@ package pre2021 import ( - "context" "testing" - "time" "github.com/gochan-org/gochan/pkg/gcsql" "github.com/stretchr/testify/assert" @@ -26,7 +24,23 @@ func TestMigrateBoardsToNewDB(t *testing.T) { if !assert.NoError(t, migrator.MigrateBoards()) { t.FailNow() } + validateBoardMigration(t) +} +func TestMigrateBoardsInPlace(t *testing.T) { + outDir := t.TempDir() + migrator := setupMigrationTest(t, outDir, true) + if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { + t.FailNow() + } + + if !assert.NoError(t, migrator.MigrateBoards()) { + t.FailNow() + } + validateBoardMigration(t) +} + +func validateBoardMigration(t *testing.T) { migratedBoards, err := gcsql.GetAllBoards(false) if !assert.NoError(t, err) { t.FailNow() @@ -51,7 +65,7 @@ func TestMigrateBoardsToNewDB(t *testing.T) { if !assert.NoError(t, err) { t.FailNow() } - assert.Equal(t, 1, testBoard.ID) + assert.Greater(t, testBoard.ID, 0) assert.Equal(t, "Testing Board", testBoard.Title) assert.Equal(t, "Board for testing pre-2021 migration", testBoard.Subtitle) assert.Equal(t, "Board for testing pre-2021 migration description", testBoard.Description) @@ -67,24 +81,3 @@ func TestMigrateBoardsToNewDB(t *testing.T) { } assert.Equal(t, "Hidden Board", hiddenBoard.Title) } - -func TestMigrateBoardsInPlace(t *testing.T) { - outDir := t.TempDir() - migrator := setupMigrationTest(t, outDir, true) - if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigrateBoards()) { - t.FailNow() - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(migrator.config.DBTimeoutSeconds)) - defer cancel() - var uri string - var sectionID int - assert.NoError(t, migrator.db.QueryRowContextSQL(ctx, nil, "SELECT uri FROM DBPREFIXboards WHERE dir = ?", []any{"test"}, []any{&uri})) - assert.NoError(t, migrator.db.QueryRowContextSQL(ctx, nil, "SELECT section_id FROM DBPREFIXboards WHERE dir = ?", []any{"test"}, []any{§ionID})) - assert.Equal(t, "", uri) - assert.Greater(t, sectionID, 0) -} diff --git a/cmd/gochan-migration/internal/pre2021/posts_test.go b/cmd/gochan-migration/internal/pre2021/posts_test.go index 34f38624..160a5dae 100644 --- a/cmd/gochan-migration/internal/pre2021/posts_test.go +++ b/cmd/gochan-migration/internal/pre2021/posts_test.go @@ -27,27 +27,7 @@ func TestMigratePostsToNewDB(t *testing.T) { if !assert.NoError(t, migrator.MigratePosts()) { t.FailNow() } - - var numMigratedThreads int - if !assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numMigratedThreads}), "Failed to get number of migrated threads") { - t.FailNow() - } - assert.Equal(t, 2, numMigratedThreads, "Expected to have three migrated threads") - - var locked bool - if !assert.NoError(t, gcsql.QueryRowSQL("SELECT locked FROM DBPREFIXthreads WHERE id = 1", nil, []any{&locked})) { - t.FailNow() - } - assert.True(t, locked, "Expected thread ID 1 to be locked") - - // make sure deleted posts and threads weren't migrated - var numDeleted int - assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXposts WHERE message_raw LIKE '%deleted%' OR is_deleted", nil, []any{&numDeleted})) - assert.Zero(t, numDeleted, "Expected no deleted threads to be migrated") - - var numUploadPosts int - assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXfiles", nil, []any{&numUploadPosts})) - assert.Equal(t, 1, numUploadPosts, "Expected to have 1 upload post") + validatePostMigration(t) } func TestMigratePostsInPlace(t *testing.T) { @@ -64,6 +44,10 @@ func TestMigratePostsInPlace(t *testing.T) { if !assert.NoError(t, migrator.MigratePosts()) { t.FailNow() } + validatePostMigration(t) +} + +func validatePostMigration(t *testing.T) { var numThreads int if !assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numThreads}), "Failed to get number of threads") { t.FailNow() @@ -77,4 +61,16 @@ func TestMigratePostsInPlace(t *testing.T) { var ip string assert.NoError(t, gcsql.QueryRowSQL("SELECT IP_NTOA FROM DBPREFIXposts WHERE id = 1", nil, []any{&ip})) assert.Equal(t, "192.168.56.1", ip, "Expected to have the correct IP address") + + var numMigratedThreads int + if !assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numMigratedThreads}), "Failed to get number of migrated threads") { + t.FailNow() + } + assert.Equal(t, 2, numMigratedThreads, "Expected to have three migrated threads") + + var locked bool + if !assert.NoError(t, gcsql.QueryRowSQL("SELECT locked FROM DBPREFIXthreads WHERE id = 1", nil, []any{&locked})) { + t.FailNow() + } + assert.True(t, locked, "Expected thread ID 1 to be locked") } diff --git a/cmd/gochan-migration/internal/pre2021/pre2021_test.go b/cmd/gochan-migration/internal/pre2021/pre2021_test.go index 338b0c31..e7537157 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021_test.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021_test.go @@ -99,6 +99,11 @@ func TestPre2021MigrationToNewDB(t *testing.T) { migrated, err := migrator.MigrateDB() assert.NoError(t, err) assert.False(t, migrated) + + validateBoardMigration(t) + validatePostMigration(t) + validateBanMigration(t) + validateStaffMigration(t) } func TestPre2021MigrationInPlace(t *testing.T) { @@ -110,4 +115,9 @@ func TestPre2021MigrationInPlace(t *testing.T) { migrated, err := migrator.MigrateDB() assert.NoError(t, err) assert.False(t, migrated) + + validateBoardMigration(t) + validatePostMigration(t) + validateBanMigration(t) + validateStaffMigration(t) } diff --git a/cmd/gochan-migration/internal/pre2021/staff_test.go b/cmd/gochan-migration/internal/pre2021/staff_test.go index 90c4efbf..22061a2a 100644 --- a/cmd/gochan-migration/internal/pre2021/staff_test.go +++ b/cmd/gochan-migration/internal/pre2021/staff_test.go @@ -21,11 +21,7 @@ func TestMigrateStaffToNewDB(t *testing.T) { if !assert.NoError(t, migrator.MigrateStaff()) { t.FailNow() } - migratedAdmin, err := gcsql.GetStaffByUsername("migratedadmin", true) - if !assert.NoError(t, err) { - t.FailNow() - } - assert.Equal(t, 3, migratedAdmin.Rank) + validateStaffMigration(t) } func TestMigrateStaffInPlace(t *testing.T) { @@ -38,6 +34,10 @@ func TestMigrateStaffInPlace(t *testing.T) { if !assert.NoError(t, migrator.MigrateStaff()) { t.FailNow() } + validateStaffMigration(t) +} + +func validateStaffMigration(t *testing.T) { migratedAdmin, err := gcsql.GetStaffByUsername("migratedadmin", true) if !assert.NoError(t, err) { t.FailNow() From 9298aa42c30d6fffb76d751548230e0f2e752c1b Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 2 Feb 2025 15:07:47 -0800 Subject: [PATCH 090/122] Return error if section doesn't exist while updating --- pkg/gcsql/sections.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/gcsql/sections.go b/pkg/gcsql/sections.go index 18ab4037..1a82a74b 100644 --- a/pkg/gcsql/sections.go +++ b/pkg/gcsql/sections.go @@ -148,7 +148,15 @@ func NewSection(name string, abbreviation string, hidden bool, position int) (*S } func (s *Section) UpdateValues() error { + var count int + err := QueryRowTimeoutSQL(nil, `SELECT COUNT(*) FROM DBPREFIXsections WHERE id = ?`, []any{s.ID}, []any{&count}) + if errors.Is(err, sql.ErrNoRows) { + return ErrSectionDoesNotExist + } + if err != nil { + return err + } const query = `UPDATE DBPREFIXsections set name = ?, abbreviation = ?, position = ?, hidden = ? WHERE id = ?` - _, err := ExecTimeoutSQL(nil, query, s.Name, s.Abbreviation, s.Position, s.Hidden, s.ID) + _, err = ExecTimeoutSQL(nil, query, s.Name, s.Abbreviation, s.Position, s.Hidden, s.ID) return err } From cbd1dd8a99c2499610a4973ce206d8936601b010 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 2 Feb 2025 15:26:33 -0800 Subject: [PATCH 091/122] Start changing migrating to have "in-place" migration rename boards to be recreated and populated --- .../internal/pre2021/announcements.go | 65 +------ .../internal/pre2021/announcements_test.go | 27 +-- .../internal/pre2021/bans_test.go | 28 +-- .../internal/pre2021/boards.go | 103 +++-------- .../internal/pre2021/boards_test.go | 15 +- .../internal/pre2021/posts.go | 163 +----------------- .../internal/pre2021/posts_test.go | 19 +- .../internal/pre2021/pre2021.go | 47 ++++- .../internal/pre2021/pre2021_test.go | 8 +- .../internal/pre2021/queries.go | 26 +-- .../internal/pre2021/staff.go | 88 ++++------ .../internal/pre2021/staff_test.go | 15 +- 12 files changed, 127 insertions(+), 477 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/announcements.go b/cmd/gochan-migration/internal/pre2021/announcements.go index e015f934..ab6ce9bc 100644 --- a/cmd/gochan-migration/internal/pre2021/announcements.go +++ b/cmd/gochan-migration/internal/pre2021/announcements.go @@ -8,63 +8,7 @@ import ( "github.com/gochan-org/gochan/pkg/gcsql" ) -func (m *Pre2021Migrator) migrateAnnouncementsInPlace() error { - errEv := common.LogError() - defer errEv.Discard() - - if _, err := gcsql.ExecSQL(announcementsAlterStatement); err != nil { - errEv.Err(err).Caller().Msg("Failed to alter announcements table") - return err - } - - var err error - m.migrationUser, err = m.getMigrationUser(errEv) - if err != nil { - errEv.Err(err).Caller().Msg("Failed to get migration user") - return err - } - - rows, err := m.db.QuerySQL("SELECT poster FROM DBPREFIXannouncements") - if err != nil { - errEv.Err(err).Caller().Msg("Failed to get announcements") - return err - } - defer rows.Close() - - var announcementPosters []string - for rows.Next() { - var poster string - if err = rows.Scan(&poster); err != nil { - errEv.Err(err).Caller().Msg("Failed to scan announcement row") - return err - } - announcementPosters = append(announcementPosters, poster) - } - if err = rows.Close(); err != nil { - errEv.Err(err).Caller().Msg("Failed to close announcement rows") - return err - } - for _, poster := range announcementPosters { - id, err := gcsql.GetStaffID(poster) - if errors.Is(err, gcsql.ErrUnrecognizedUsername) { - // user doesn't exist, use migration user - common.LogWarning().Str("staff", poster).Msg("Staff username not found in database") - id = m.migrationUser.ID - } else if err != nil { - errEv.Err(err).Caller().Str("staff", poster).Msg("Failed to get staff ID") - return err - } - - if _, err = gcsql.ExecSQL("UPDATE DBPREFIXannouncements SET staff_id = ? WHERE poster = ?", id, poster); err != nil { - errEv.Err(err).Caller().Str("staff", poster).Msg("Failed to update announcement poster") - return err - } - } - - return nil -} - -func (m *Pre2021Migrator) migrateAnnouncementsToNewDB() error { +func (m *Pre2021Migrator) MigrateAnnouncements() error { errEv := common.LogError() defer errEv.Discard() @@ -112,10 +56,3 @@ func (m *Pre2021Migrator) migrateAnnouncementsToNewDB() error { } return nil } - -func (m *Pre2021Migrator) MigrateAnnouncements() error { - if m.IsMigratingInPlace() { - return m.migrateAnnouncementsInPlace() - } - return m.migrateAnnouncementsToNewDB() -} diff --git a/cmd/gochan-migration/internal/pre2021/announcements_test.go b/cmd/gochan-migration/internal/pre2021/announcements_test.go index 6cb4b2da..99684351 100644 --- a/cmd/gochan-migration/internal/pre2021/announcements_test.go +++ b/cmd/gochan-migration/internal/pre2021/announcements_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMigrateAnnouncementsToNewDB(t *testing.T) { +func TestMigrateAnnouncements(t *testing.T) { outDir := t.TempDir() migrator := setupMigrationTest(t, outDir, false) if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { @@ -26,31 +26,6 @@ func TestMigrateAnnouncementsToNewDB(t *testing.T) { t.FailNow() } - validateAnnouncementMigration(t) -} - -func TestMigrateAnnouncementsInPlace(t *testing.T) { - outDir := t.TempDir() - migrator := setupMigrationTest(t, outDir, true) - if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigrateBoards()) { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigrateStaff()) { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigrateAnnouncements()) { - t.FailNow() - } - validateAnnouncementMigration(t) -} - -func validateAnnouncementMigration(t *testing.T) { var numAnnouncements int assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXannouncements WHERE staff_id > 0", nil, []any{&numAnnouncements})) assert.Equal(t, 2, numAnnouncements, "Expected to have two announcement") diff --git a/cmd/gochan-migration/internal/pre2021/bans_test.go b/cmd/gochan-migration/internal/pre2021/bans_test.go index 3efa9412..43f0a737 100644 --- a/cmd/gochan-migration/internal/pre2021/bans_test.go +++ b/cmd/gochan-migration/internal/pre2021/bans_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMigrateBansToNewDB(t *testing.T) { +func TestMigrateBans(t *testing.T) { outDir := t.TempDir() migrator := setupMigrationTest(t, outDir, false) if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { @@ -33,32 +33,6 @@ func TestMigrateBansToNewDB(t *testing.T) { validateBanMigration(t) } -func TestMigrateBansInPlace(t *testing.T) { - outDir := t.TempDir() - migrator := setupMigrationTest(t, outDir, true) - if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigrateBoards()) { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigratePosts()) { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigrateStaff()) { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigrateBans()) { - t.FailNow() - } - - validateBanMigration(t) -} - func validateBanMigration(t *testing.T) { bans, err := gcsql.GetIPBans(0, 200, false) if !assert.NoError(t, err) { diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index ce1f2e83..6b204ad8 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -1,13 +1,9 @@ package pre2021 import ( - "runtime/debug" - "strings" - "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/config" "github.com/gochan-org/gochan/pkg/gcsql" - "github.com/rs/zerolog" ) type migrationBoard struct { @@ -21,41 +17,7 @@ type migrationSection struct { gcsql.Section } -func (m *Pre2021Migrator) migrateSectionsInPlace() error { - _, err := m.db.ExecSQL(`ALTER TABLE DBPREFIXsections RENAME COLUMN list_order TO position`) - if err != nil { - common.LogError().Caller().Msg("Failed to rename list_order column to position") - } - return err -} - -func (m *Pre2021Migrator) migrateBoardsInPlace() error { - errEv := common.LogError() - defer errEv.Discard() - err := m.migrateSectionsInPlace() - if err != nil { - errEv.Err(err).Caller().Msg("Failed to migrate sections") - return err - } - - for _, statement := range boardAlterStatements { - if strings.Contains(statement, "CONSTRAINT") && m.db.SQLDriver() == "sqlite3" { - // skip constraints in SQLite since they can't be added after table creation - continue - } - _, err = m.db.ExecSQL(statement) - if err != nil { - errEv.Err(err).Caller(). - Str("statement", statement). - Msg("Failed to execute alter statement") - return err - } - } - - return nil -} - -func (m *Pre2021Migrator) migrateSectionsToNewDB() error { +func (m *Pre2021Migrator) migrateSections() error { // creates sections in the new db if they don't exist, and also creates a migration section that // boards will be set to, to be moved to the correct section by the admin after migration errEv := common.LogError() @@ -74,6 +36,7 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error { }) } + var sectionsToBeCreated []gcsql.Section rows, err := m.db.QuerySQL(sectionsQuery) if err != nil { errEv.Err(err).Caller().Msg("Failed to query old database sections") @@ -99,33 +62,40 @@ func (m *Pre2021Migrator) migrateSectionsToNewDB() error { Int("oldSectionID", m.sections[s].oldID). Str("sectionName", section.Name). Str("sectionAbbreviation", section.Abbreviation). - Msg("Section already exists in new db, updating values") - if err = m.sections[s].UpdateValues(); err != nil { - errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to update pre-existing section values") - } + Msg("Section already exists in new db, values will be updated") found = true break } } if !found { - migratedSection, err := gcsql.NewSection(section.Name, section.Abbreviation, section.Hidden, section.Position) - if err != nil { - errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to migrate section") - return err - } - m.sections = append(m.sections, migrationSection{ - Section: *migratedSection, - }) + sectionsToBeCreated = append(sectionsToBeCreated, section) } } if err = rows.Close(); err != nil { errEv.Caller().Msg("Failed to close section rows") return err } + for _, section := range sectionsToBeCreated { + migratedSection, err := gcsql.NewSection(section.Name, section.Abbreviation, section.Hidden, section.Position) + if err != nil { + errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to migrate section") + return err + } + m.sections = append(m.sections, migrationSection{ + Section: *migratedSection, + }) + } + + for s, section := range m.sections { + if err = m.sections[s].UpdateValues(); err != nil { + errEv.Err(err).Caller().Str("sectionName", section.Name).Msg("Failed to update pre-existing section values") + } + } + return nil } -func (m *Pre2021Migrator) migrateBoardsToNewDB() error { +func (m *Pre2021Migrator) MigrateBoards() error { m.boards = nil errEv := common.LogError() defer errEv.Discard() @@ -137,7 +107,7 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { return nil } - if err = m.migrateSectionsToNewDB(); err != nil { + if err = m.migrateSections(); err != nil { // error should already be logged by migrateSectionsToNewDB return err } @@ -162,6 +132,7 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { return err } defer rows.Close() + var boardsTmp []migrationBoard for rows.Next() { var board migrationBoard @@ -176,8 +147,11 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { return err } board.MaxThreads = maxPages * config.GetBoardConfig(board.Dir).ThreadsPerPage - found := false + boardsTmp = append(boardsTmp, board) + } + for _, board := range boardsTmp { + found := false for b, newBoard := range m.boards { if newBoard.Dir == board.Dir { m.boards[b].oldID = board.oldID @@ -229,24 +203,3 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error { } return nil } - -func (m *Pre2021Migrator) MigrateBoards() error { - defer func() { - if r := recover(); r != nil { - stackTrace := debug.Stack() - traceLines := strings.Split(string(stackTrace), "\n") - zlArr := zerolog.Arr() - for _, line := range traceLines { - zlArr.Str(line) - } - common.LogFatal().Caller(). - Interface("recover", r). - Array("stackTrace", zlArr). - Msg("Recovered from panic in MigrateBoards") - } - }() - if m.IsMigratingInPlace() { - return m.migrateBoardsInPlace() - } - return m.migrateBoardsToNewDB() -} diff --git a/cmd/gochan-migration/internal/pre2021/boards_test.go b/cmd/gochan-migration/internal/pre2021/boards_test.go index d42152b1..d636a58b 100644 --- a/cmd/gochan-migration/internal/pre2021/boards_test.go +++ b/cmd/gochan-migration/internal/pre2021/boards_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMigrateBoardsToNewDB(t *testing.T) { +func TestMigrateBoards(t *testing.T) { outDir := t.TempDir() migrator := setupMigrationTest(t, outDir, false) if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { @@ -27,19 +27,6 @@ func TestMigrateBoardsToNewDB(t *testing.T) { validateBoardMigration(t) } -func TestMigrateBoardsInPlace(t *testing.T) { - outDir := t.TempDir() - migrator := setupMigrationTest(t, outDir, true) - if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigrateBoards()) { - t.FailNow() - } - validateBoardMigration(t) -} - func validateBoardMigration(t *testing.T) { migratedBoards, err := gcsql.GetAllBoards(false) if !assert.NoError(t, err) { diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index ea8b8f96..bd4bfa56 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -3,8 +3,6 @@ package pre2021 import ( "context" "database/sql" - "os" - "strings" "time" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" @@ -35,13 +33,6 @@ type migrationPost struct { oldParentID int } -func (m *Pre2021Migrator) MigratePosts() error { - if m.IsMigratingInPlace() { - return m.migratePostsInPlace() - } - return m.migratePostsToNewDB() -} - func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *zerolog.Event) error { var err error @@ -83,7 +74,7 @@ func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *ze return nil } -func (m *Pre2021Migrator) migratePostsToNewDB() error { +func (m *Pre2021Migrator) MigratePosts() error { errEv := common.LogError() defer errEv.Discard() @@ -195,155 +186,3 @@ func (m *Pre2021Migrator) migratePostsToNewDB() error { Msg("Migrated threads successfully") return nil } - -func (m *Pre2021Migrator) migratePostsInPlace() error { - errEv := common.LogError() - defer errEv.Discard() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(m.config.DBTimeoutSeconds)) - defer cancel() - - ba, err := os.ReadFile(gcutil.FindResource("sql/initdb_" + m.db.SQLDriver() + ".sql")) - if err != nil { - errEv.Err(err).Caller(). - Msg("Failed to read initdb SQL file") - return err - } - statements := strings.Split(string(ba), ";") - for _, statement := range statements { - statement = strings.TrimSpace(statement) - if strings.HasPrefix(statement, "CREATE TABLE DBPREFIXthreads") || strings.HasPrefix(statement, "CREATE TABLE DBPREFIXfiles") { - if _, err = m.db.ExecContextSQL(ctx, nil, statement); err != nil { - errEv.Err(err).Caller().Msg("Failed to create threads table") - return err - } - } - } - - rows, err := m.db.QueryContextSQL(ctx, nil, threadsQuery+" AND parentid = 0") - if err != nil { - errEv.Err(err).Caller(). - Msg("Failed to get threads") - return err - } - defer rows.Close() - - var threads []migrationPost - for rows.Next() { - var post migrationPost - if err = rows.Scan( - &post.ID, &post.oldBoardID, &post.oldParentID, &post.Name, &post.Tripcode, &post.Email, - &post.Subject, &post.Message, &post.MessageRaw, &post.Password, &post.filename, - &post.filenameOriginal, &post.fileChecksum, &post.filesize, &post.imageW, &post.imageH, - &post.thumbW, &post.thumbH, &post.IP, &post.CreatedOn, &post.autosage, - &post.bumped, &post.stickied, &post.locked, - ); err != nil { - errEv.Err(err).Caller(). - Msg("Failed to scan thread") - return err - } - threads = append(threads, post) - } - if err = rows.Close(); err != nil { - errEv.Caller().Msg("Failed to close thread rows") - return err - } - - for _, statements := range postAlterStatements { - if _, err = m.db.ExecContextSQL(ctx, nil, statements); err != nil { - errEv.Err(err).Caller().Msg("Failed to alter posts table") - return err - } - } - - switch m.db.SQLDriver() { - case "mysql": - _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts ADD COLUMN ip_new VARBINARY(16) NOT NULL") - case "postgres", "postgresql": - _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts ADD COLUMN ip_new INET NOT NULL") - case "sqlite3": - _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts ADD COLUMN ip_new VARCHAR(45) NOT NULL DEFAULT '0.0.0.0'") - } - if err != nil { - errEv.Err(err).Caller().Msg("Failed to update IP column") - return err - } - if _, err = m.db.ExecContextSQL(ctx, nil, "UPDATE DBPREFIXposts SET ip_new = IP_ATON"); err != nil { - errEv.Err(err).Caller().Msg("Failed to update IP column") - return err - } - if _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts RENAME COLUMN ip TO ip_old"); err != nil { - errEv.Err(err).Caller().Msg("Failed to rename old IP column") - return err - } - if _, err = m.db.ExecContextSQL(ctx, nil, "ALTER TABLE DBPREFIXposts RENAME COLUMN ip_new TO ip"); err != nil { - errEv.Err(err).Caller().Msg("Failed to rename new IP column") - return err - } - - for _, op := range threads { - if _, err = m.db.ExecContextSQL(ctx, nil, - `INSERT INTO DBPREFIXthreads(board_id,locked,stickied,anchored,cyclical,last_bump,is_deleted) VALUES(?,?,?,?,?,?,?)`, - op.oldBoardID, op.locked, op.stickied, op.autosage, false, op.bumped, false, - ); err != nil { - errEv.Err(err).Caller(). - Int("postID", op.ID). - Msg("Failed to insert thread") - return err - } - if err = m.db.QueryRowContextSQL(ctx, nil, "SELECT MAX(id) FROM DBPREFIXthreads", nil, []any{&op.ThreadID}); err != nil { - errEv.Err(err).Caller(). - Int("postID", op.ID). - Msg("Failed to get thread ID") - return err - } - - if _, err = m.db.ExecContextSQL(ctx, nil, - "UPDATE DBPREFIXposts SET thread_id = ? WHERE (id = ? and is_top_post) or thread_id = ?", op.ThreadID, op.oldID, op.oldID, - ); err != nil { - errEv.Err(err).Caller(). - Int("postID", op.ID). - Int("threadID", op.ThreadID). - Msg("Failed to set thread ID") - return err - } - } - if rows, err = m.db.QueryContextSQL(ctx, nil, - "SELECT id,filename,filename_original,file_checksum,filesize,image_w,image_h,thumb_w,thumb_h FROM DBPREFIXposts WHERE filename <> ''", - ); err != nil { - errEv.Err(err).Caller().Msg("Failed to get uploads") - return err - } - defer rows.Close() - - var uploads []gcsql.Upload - for rows.Next() { - var upload gcsql.Upload - if err = rows.Scan(&upload.PostID, &upload.Filename, &upload.OriginalFilename, &upload.Checksum, &upload.FileSize, &upload.Width, - &upload.Height, &upload.ThumbnailWidth, &upload.ThumbnailHeight, - ); err != nil { - errEv.Err(err).Caller().Msg("Failed to scan upload") - return err - } - uploads = append(uploads, upload) - } - if err = rows.Close(); err != nil { - errEv.Caller().Msg("Failed to close upload rows") - return err - } - - for _, upload := range uploads { - if _, err = m.db.ExecContextSQL(ctx, nil, - `INSERT INTO DBPREFIXfiles(post_id,file_order,filename,original_filename,checksum,file_size,width,height,thumbnail_width,thumbnail_height,is_spoilered) VALUES - (?,0,?,?,?,?,?,?,?,?,0)`, - upload.PostID, upload.Filename, upload.OriginalFilename, upload.Checksum, upload.FileSize, upload.Width, upload.Height, - upload.ThumbnailWidth, upload.ThumbnailHeight, - ); err != nil { - errEv.Err(err).Caller(). - Int("postID", upload.PostID). - Msg("Failed to insert upload") - return err - } - } - - return nil -} diff --git a/cmd/gochan-migration/internal/pre2021/posts_test.go b/cmd/gochan-migration/internal/pre2021/posts_test.go index 160a5dae..f4b605b2 100644 --- a/cmd/gochan-migration/internal/pre2021/posts_test.go +++ b/cmd/gochan-migration/internal/pre2021/posts_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMigratePostsToNewDB(t *testing.T) { +func TestMigratePosts(t *testing.T) { outDir := t.TempDir() migrator := setupMigrationTest(t, outDir, false) if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { @@ -30,23 +30,6 @@ func TestMigratePostsToNewDB(t *testing.T) { validatePostMigration(t) } -func TestMigratePostsInPlace(t *testing.T) { - outDir := t.TempDir() - migrator := setupMigrationTest(t, outDir, true) - if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigrateBoards()) { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigratePosts()) { - t.FailNow() - } - validatePostMigration(t) -} - func validatePostMigration(t *testing.T) { var numThreads int if !assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numThreads}), "Failed to get number of threads") { diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index d548f875..8562569e 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -4,6 +4,7 @@ package pre2021 import ( "context" "encoding/json" + "fmt" "os" "time" @@ -25,6 +26,7 @@ type Pre2021Migrator struct { migrationUser *gcsql.Staff boards []migrationBoard sections []migrationSection + staff []migrationStaff } // IsMigratingInPlace implements common.DBMigrator. @@ -45,7 +47,7 @@ func (m *Pre2021Migrator) readConfig() error { func (m *Pre2021Migrator) Init(options *common.MigrationOptions) error { m.options = options var err error - m.config.SQLConfig = config.GetSQLConfig() + if err = m.readConfig(); err != nil { return err } @@ -66,6 +68,43 @@ func (m *Pre2021Migrator) IsMigrated() (bool, error) { return common.TableExists(ctx, m.db, nil, "DBPREFIXdatabase_version", &sqlConfig) } +func (m *Pre2021Migrator) renameTablesForInPlace() error { + var err error + errEv := common.LogError() + defer errEv.Discard() + if _, err = m.db.ExecSQL("DROP TABLE DBPREFIXinfo"); err != nil { + errEv.Err(err).Caller().Msg("Error dropping info table") + return err + } + for _, table := range renameTables { + if _, err = m.db.ExecSQL(fmt.Sprintf(renameTableStatementTemplate, table, table)); err != nil { + errEv.Caller().Err(err). + Str("table", table). + Msg("Error renaming table") + return err + } + } + + if err = gcsql.CheckAndInitializeDatabase(m.config.DBtype, "4"); err != nil { + errEv.Caller().Err(err).Msg("Error checking and initializing database") + return err + } + + if err = m.Close(); err != nil { + errEv.Err(err).Caller().Msg("Error closing database") + return err + } + m.config.SQLConfig.DBprefix = "_tmp_" + m.config.DBprefix + m.db, err = gcsql.Open(&m.config.SQLConfig) + if err != nil { + errEv.Err(err).Caller().Msg("Error reopening database with new prefix") + return err + } + + common.LogInfo().Msg("Renamed tables for in-place migration") + return err +} + func (m *Pre2021Migrator) MigrateDB() (bool, error) { errEv := common.LogError() defer errEv.Discard() @@ -78,6 +117,12 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) { return true, nil } + if m.IsMigratingInPlace() { + if err = m.renameTablesForInPlace(); err != nil { + return false, err + } + } + if err := m.MigrateBoards(); err != nil { return false, err } diff --git a/cmd/gochan-migration/internal/pre2021/pre2021_test.go b/cmd/gochan-migration/internal/pre2021/pre2021_test.go index e7537157..d424eb53 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021_test.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021_test.go @@ -97,7 +97,9 @@ func TestPre2021MigrationToNewDB(t *testing.T) { t.FailNow() } migrated, err := migrator.MigrateDB() - assert.NoError(t, err) + if !assert.NoError(t, err) { + t.FailNow() + } assert.False(t, migrated) validateBoardMigration(t) @@ -113,7 +115,9 @@ func TestPre2021MigrationInPlace(t *testing.T) { t.FailNow() } migrated, err := migrator.MigrateDB() - assert.NoError(t, err) + if !assert.NoError(t, err) { + t.FailNow() + } assert.False(t, migrated) validateBoardMigration(t) diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index 4342446b..d233d01d 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -14,14 +14,14 @@ bumped, stickied, locked FROM DBPREFIXposts WHERE deleted_timestamp IS NULL` threadsQuery = postsQuery + " AND parentid = 0" - staffQuery = `SELECT username, rank, boards, added_on, last_active FROM DBPREFIXstaff` + staffQuery = `SELECT id, username, rank, boards, added_on, last_active FROM DBPREFIXstaff` bansQuery = `SELECT id, allow_read, COALESCE(ip, '') as ip, name, name_is_regex, filename, file_checksum, boards, staff, timestamp, expires, permaban, reason, type, staff_note, appeal_at, can_appeal FROM DBPREFIXbanlist` announcementsQuery = "SELECT id, subject, message, poster, timestamp FROM DBPREFIXannouncements" - announcementsAlterStatement = "ALTER TABLE DBPREFIXannouncements ADD COLUMN staff_id INT NOT NULL DEFAULT 1" + renameTableStatementTemplate = "ALTER TABLE %s RENAME TO _tmp_%s" ) var ( @@ -41,22 +41,10 @@ var ( "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_dir_unique UNIQUE (dir)", "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_uri_unique UNIQUE (uri)", } - postAlterStatements = []string{ - "ALTER TABLE DBPREFIXposts RENAME COLUMN parentid TO thread_id", - "ALTER TABLE DBPREFIXposts RENAME COLUMN timestamp TO created_on", - "ALTER TABLE DBPREFIXposts RENAME COLUMN deleted_timestamp TO deleted_at", - // "ALTER TABLE DBPREFIXposts RENAME COLUMN ip TO ip_old", - "ALTER TABLE DBPREFIXposts ADD COLUMN is_top_post BOOL NOT NULL DEFAULT FALSE", - "ALTER TABLE DBPREFIXposts ADD COLUMN is_role_signature BOOL NOT NULL DEFAULT FALSE", - "ALTER TABLE DBPREFIXposts ADD COLUMN is_deleted BOOL NOT NULL DEFAULT FALSE", - "ALTER TABLE DBPREFIXposts ADD COLUMN banned_message TEXT", - "ALTER TABLE DBPREFIXposts ADD COLUMN flag VARCHAR(45) NOT NULL DEFAULT ''", - "ALTER TABLE DBPREFIXposts ADD COLUMN country VARCHAR(80) NOT NULL DEFAULT ''", - "UPDATE DBPREFIXposts SET is_top_post = TRUE WHERE thread_id = 0", - } - staffAlterStatements = []string{ - "ALTER TABLE DBPREFIXstaff RENAME COLUMN rank TO global_rank", - "ALTER TABLE DBPREFIXstaff RENAME COLUMN last_active TO last_login", - "ALTER TABLE DBPREFIXstaff ADD COLUMN is_active BOOL NOT NULL DEFAULT TRUE", + + // tables to be renamed to _tmp_DBPREFIX* to work around SQLite's lack of support for changing/removing columns + renameTables = []string{ + "DBPREFIXannouncements", "DBPREFIXappeals", "DBPREFIXbanlist", "DBPREFIXboards", "DBPREFIXembeds", "DBPREFIXlinks", + "DBPREFIXposts", "DBPREFIXreports", "DBPREFIXsections", "DBPREFIXsessions", "DBPREFIXstaff", "DBPREFIXwordfilters", } ) diff --git a/cmd/gochan-migration/internal/pre2021/staff.go b/cmd/gochan-migration/internal/pre2021/staff.go index cca7d9cd..f1333743 100644 --- a/cmd/gochan-migration/internal/pre2021/staff.go +++ b/cmd/gochan-migration/internal/pre2021/staff.go @@ -11,23 +11,10 @@ import ( "github.com/rs/zerolog" ) -func (m *Pre2021Migrator) migrateStaffInPlace() error { - errEv := common.LogError() - defer errEv.Discard() - - for _, stmt := range staffAlterStatements { - if _, err := gcsql.ExecSQL(stmt); err != nil { - errEv.Err(err).Caller().Msg("Failed to alter staff table") - return err - } - } - - _, err := m.getMigrationUser(errEv) - if err != nil { - return err - } - - return nil +type migrationStaff struct { + gcsql.Staff + boards string + oldID int } func (m *Pre2021Migrator) getMigrationUser(errEv *zerolog.Event) (*gcsql.Staff, error) { @@ -36,7 +23,7 @@ func (m *Pre2021Migrator) getMigrationUser(errEv *zerolog.Event) (*gcsql.Staff, } user := &gcsql.Staff{ - Username: "pre2021-migration" + gcutil.RandomString(15), + Username: "pre2021-migration" + gcutil.RandomString(8), AddedOn: time.Now(), } _, err := gcsql.ExecSQL("INSERT INTO DBPREFIXstaff(username,password_checksum,global_rank,is_active) values(?,'',0,0)", user.Username) @@ -50,10 +37,11 @@ func (m *Pre2021Migrator) getMigrationUser(errEv *zerolog.Event) (*gcsql.Staff, return nil, err } m.migrationUser = user + m.staff = append(m.staff, migrationStaff{Staff: *user}) return user, nil } -func (m *Pre2021Migrator) migrateStaffToNewDB() error { +func (m *Pre2021Migrator) MigrateStaff() error { errEv := common.LogError() defer errEv.Discard() @@ -70,56 +58,53 @@ func (m *Pre2021Migrator) migrateStaffToNewDB() error { defer rows.Close() for rows.Next() { - var username string - var rank int - var boards string - var addedOn, lastActive time.Time - - if err = rows.Scan(&username, &rank, &boards, &addedOn, &lastActive); err != nil { + var staff migrationStaff + if err = rows.Scan(&staff.oldID, &staff.Username, &staff.Rank, &staff.boards, &staff.AddedOn, &staff.LastLogin); err != nil { errEv.Err(err).Caller().Msg("Failed to scan staff row") return err } - _, err = gcsql.GetStaffByUsername(username, false) + m.staff = append(m.staff, staff) + } + for _, staff := range m.staff { + newStaff, err := gcsql.GetStaffByUsername(staff.Username, false) if err == nil { // found staff - gcutil.LogInfo().Str("username", username).Int("rank", rank).Msg("Found matching staff account") - } - if errors.Is(err, gcsql.ErrUnrecognizedUsername) { + gcutil.LogInfo().Str("username", staff.Username).Int("rank", staff.Rank).Msg("Found matching staff account") + staff.ID = newStaff.ID + } else if errors.Is(err, gcsql.ErrUnrecognizedUsername) { // staff doesn't exist, create it (with invalid checksum to be updated by the admin) - if _, err2 := gcsql.ExecSQL( + if _, err := gcsql.ExecSQL( "INSERT INTO DBPREFIXstaff(username,password_checksum,global_rank,added_on,last_login,is_active) values(?,'',?,?,?,1)", - username, rank, addedOn, lastActive, - ); err2 != nil { - errEv.Err(err2).Caller(). - Str("username", username).Int("rank", rank). - Msg("Failed to migrate staff account") + staff.Username, staff.Rank, staff.AddedOn, staff.LastLogin, + ); err != nil { + errEv.Err(err).Caller().Str("username", staff.Username).Int("rank", staff.Rank).Msg("Failed to migrate staff account") return err } - gcutil.LogInfo().Str("username", username).Int("rank", rank).Msg("Successfully migrated staff account") - } else if err != nil { - errEv.Err(err).Caller().Str("username", username).Msg("Failed to get staff account info") + if staff.ID, err = gcsql.GetStaffID(staff.Username); err != nil { + errEv.Err(err).Caller().Str("username", staff.Username).Msg("Failed to get staff account ID") + return err + } + gcutil.LogInfo().Str("username", staff.Username).Int("rank", staff.Rank).Msg("Successfully migrated staff account") + } else { + errEv.Err(err).Caller().Str("username", staff.Username).Msg("Failed to get staff account info") return err } - staffID, err := gcsql.GetStaffID(username) - if err != nil { - errEv.Err(err).Caller().Str("username", username).Msg("Failed to get staff account ID") - return err - } - if boards != "" && boards != "*" { - boardsArr := strings.Split(boards, ",") + + if staff.boards != "" && staff.boards != "*" { + boardsArr := strings.Split(staff.boards, ",") for _, board := range boardsArr { board = strings.TrimSpace(board) boardID, err := gcsql.GetBoardIDFromDir(board) if err != nil { errEv.Err(err).Caller(). - Str("username", username). + Str("username", staff.Username). Str("board", board). Msg("Failed to get board ID") return err } - if _, err = gcsql.ExecSQL("INSERT INTO DBPREFIXboard_staff(board_id,staff_id) VALUES(?,?)", boardID, staffID); err != nil { + if _, err = gcsql.ExecSQL("INSERT INTO DBPREFIXboard_staff(board_id,staff_id) VALUES(?,?)", boardID, staff.ID); err != nil { errEv.Err(err).Caller(). - Str("username", username). + Str("username", staff.Username). Str("board", board). Msg("Failed to apply staff board info") return err @@ -134,10 +119,3 @@ func (m *Pre2021Migrator) migrateStaffToNewDB() error { } return nil } - -func (m *Pre2021Migrator) MigrateStaff() error { - if m.IsMigratingInPlace() { - return m.migrateStaffInPlace() - } - return m.migrateStaffToNewDB() -} diff --git a/cmd/gochan-migration/internal/pre2021/staff_test.go b/cmd/gochan-migration/internal/pre2021/staff_test.go index 22061a2a..24a5913b 100644 --- a/cmd/gochan-migration/internal/pre2021/staff_test.go +++ b/cmd/gochan-migration/internal/pre2021/staff_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMigrateStaffToNewDB(t *testing.T) { +func TestMigrateStaff(t *testing.T) { outDir := t.TempDir() migrator := setupMigrationTest(t, outDir, false) if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") { @@ -24,19 +24,6 @@ func TestMigrateStaffToNewDB(t *testing.T) { validateStaffMigration(t) } -func TestMigrateStaffInPlace(t *testing.T) { - outDir := t.TempDir() - migrator := setupMigrationTest(t, outDir, true) - if !assert.True(t, migrator.IsMigratingInPlace(), "This test should be migrating in place") { - t.FailNow() - } - - if !assert.NoError(t, migrator.MigrateStaff()) { - t.FailNow() - } - validateStaffMigration(t) -} - func validateStaffMigration(t *testing.T) { migratedAdmin, err := gcsql.GetStaffByUsername("migratedadmin", true) if !assert.NoError(t, err) { From a38a519e4e22a323e81a3cd13d47588d2b40f90d Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sun, 2 Feb 2025 15:35:57 -0800 Subject: [PATCH 092/122] Fix database locked error in announcement migration --- .../internal/pre2021/announcements.go | 32 ++++++++++++------- .../internal/pre2021/queries.go | 17 ---------- pkg/gcsql/tables.go | 4 +-- 3 files changed, 22 insertions(+), 31 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/announcements.go b/cmd/gochan-migration/internal/pre2021/announcements.go index ab6ce9bc..aecbf909 100644 --- a/cmd/gochan-migration/internal/pre2021/announcements.go +++ b/cmd/gochan-migration/internal/pre2021/announcements.go @@ -2,12 +2,16 @@ package pre2021 import ( "errors" - "time" "github.com/gochan-org/gochan/cmd/gochan-migration/internal/common" "github.com/gochan-org/gochan/pkg/gcsql" ) +type migrationAnnouncement struct { + gcsql.Announcement + oldPoster string +} + func (m *Pre2021Migrator) MigrateAnnouncements() error { errEv := common.LogError() defer errEv.Discard() @@ -23,29 +27,33 @@ func (m *Pre2021Migrator) MigrateAnnouncements() error { return err } + var oldAnnouncements []migrationAnnouncement + for rows.Next() { - var id int - var subject, message, staff string - var timestamp time.Time - if err = rows.Scan(&id, &subject, &message, &staff, ×tamp); err != nil { + var announcement migrationAnnouncement + + if err = rows.Scan(&announcement.ID, &announcement.Subject, &announcement.Message, &announcement.oldPoster, &announcement.Timestamp); err != nil { errEv.Err(err).Caller().Msg("Failed to scan announcement row") return err } - staffID, err := gcsql.GetStaffID(staff) + oldAnnouncements = append(oldAnnouncements, announcement) + } + for _, announcement := range oldAnnouncements { + announcement.StaffID, err = gcsql.GetStaffID(announcement.oldPoster) if errors.Is(err, gcsql.ErrUnrecognizedUsername) { // user doesn't exist, use migration user - common.LogWarning().Str("staff", staff).Msg("Staff username not found in database") - message += "\n(originally by " + staff + ")" - staffID = m.migrationUser.ID + common.LogWarning().Str("staff", announcement.oldPoster).Msg("Staff username not found in database") + announcement.Message += "\n(originally by " + announcement.oldPoster + ")" + announcement.StaffID = m.migrationUser.ID } else if err != nil { - errEv.Err(err).Caller().Str("staff", staff).Msg("Failed to get staff ID") + errEv.Err(err).Caller().Str("staff", announcement.oldPoster).Msg("Failed to get staff ID") return err } if _, err = gcsql.ExecSQL( "INSERT INTO DBPREFIXannouncements(staff_id,subject,message,timestamp) values(?,?,?,?)", - staffID, subject, message, timestamp, + announcement.StaffID, announcement.Subject, announcement.Message, announcement.Timestamp, ); err != nil { - errEv.Err(err).Caller().Str("staff", staff).Msg("Failed to migrate announcement") + errEv.Err(err).Caller().Str("staff", announcement.oldPoster).Msg("Failed to migrate announcement") return err } } diff --git a/cmd/gochan-migration/internal/pre2021/queries.go b/cmd/gochan-migration/internal/pre2021/queries.go index d233d01d..8456dad7 100644 --- a/cmd/gochan-migration/internal/pre2021/queries.go +++ b/cmd/gochan-migration/internal/pre2021/queries.go @@ -25,23 +25,6 @@ timestamp, expires, permaban, reason, type, staff_note, appeal_at, can_appeal FR ) var ( - boardAlterStatements = []string{ - "ALTER TABLE DBPREFIXboards RENAME COLUMN section TO section_id", - "ALTER TABLE DBPREFIXboards RENAME COLUMN list_order TO navbar_position", - "ALTER TABLE DBPREFIXboards RENAME COLUMN created_on TO created_at", - "ALTER TABLE DBPREFIXboards RENAME COLUMN anonymous TO anonymous_name", - "ALTER TABLE DBPREFIXboards RENAME COLUMN forced_anon TO force_anonymous", - "ALTER TABLE DBPREFIXboards RENAME COLUMN embeds_allowed TO allow_embeds", - "ALTER TABLE DBPREFIXboards ADD COLUMN uri VARCHAR(45) NOT NULL DEFAULT ''", - "ALTER TABLE DBPREFIXboards ADD COLUMN min_message_length SMALLINT NOT NULL DEFAULT 0", - "ALTER TABLE DBPREFIXboards ADD COLUMN max_threads SMALLINT NOT NULL DEFAULT 65535", - // the following statements don't work in SQLite since it doesn't support adding foreign keys after table creation. - // "in-place" migration support for SQLite may be removed - "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_section_id_fk FOREIGN KEY (section_id) REFERENCES DBPREFIXsections(id)", - "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_dir_unique UNIQUE (dir)", - "ALTER TABLE DBPREFIXboards ADD CONSTRAINT boards_uri_unique UNIQUE (uri)", - } - // tables to be renamed to _tmp_DBPREFIX* to work around SQLite's lack of support for changing/removing columns renameTables = []string{ "DBPREFIXannouncements", "DBPREFIXappeals", "DBPREFIXbanlist", "DBPREFIXboards", "DBPREFIXembeds", "DBPREFIXlinks", diff --git a/pkg/gcsql/tables.go b/pkg/gcsql/tables.go index c0b06a28..12562822 100644 --- a/pkg/gcsql/tables.go +++ b/pkg/gcsql/tables.go @@ -13,8 +13,8 @@ import ( // table: DBPREFIXannouncements type Announcement struct { - ID uint // sql: id - StaffID uint // sql: staff_id + ID int // sql: id + StaffID int // sql: staff_id Subject string // sql: subject Message string // sql: message Timestamp time.Time // sql: timestamp From 10e0da4492b0cc3746edaef5e375a68eeb33da83 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 5 Feb 2025 17:32:10 -0800 Subject: [PATCH 093/122] Add future-proof functions using struct for context, tx, etc --- cmd/gochan-migration/internal/pre2021/bans.go | 2 +- .../internal/pre2021/boards.go | 6 +- .../internal/pre2021/posts.go | 17 +- pkg/gcsql/bans.go | 56 ++++--- pkg/gcsql/boards.go | 2 +- pkg/gcsql/database.go | 158 ++++++++---------- pkg/gcsql/posts.go | 142 +++++++++------- pkg/gcsql/preload.go | 4 +- pkg/gcsql/sections.go | 53 +++--- pkg/gcsql/setup_test.go | 2 +- pkg/gcsql/threads.go | 41 +++-- pkg/gcsql/uploads.go | 43 ++--- pkg/gcsql/util.go | 92 ++++++---- 13 files changed, 332 insertions(+), 286 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/bans.go b/cmd/gochan-migration/internal/pre2021/bans.go index a7cb25da..40d8bd5c 100644 --- a/cmd/gochan-migration/internal/pre2021/bans.go +++ b/cmd/gochan-migration/internal/pre2021/bans.go @@ -80,7 +80,7 @@ func (m *Pre2021Migrator) migrateBan(tx *sql.Tx, ban *migrationBan, boardID *int migratedBan.Message = ban.reason migratedBan.StaffID = ban.staffID migratedBan.StaffNote = ban.staffNote - if err := gcsql.NewIPBanTx(tx, migratedBan); err != nil { + if err := gcsql.NewIPBan(migratedBan, &gcsql.RequestOptions{Tx: tx}); err != nil { errEv.Err(err).Caller(). Int("oldID", ban.oldID).Msg("Failed to migrate ban") return err diff --git a/cmd/gochan-migration/internal/pre2021/boards.go b/cmd/gochan-migration/internal/pre2021/boards.go index 6b204ad8..8328bad1 100644 --- a/cmd/gochan-migration/internal/pre2021/boards.go +++ b/cmd/gochan-migration/internal/pre2021/boards.go @@ -37,7 +37,7 @@ func (m *Pre2021Migrator) migrateSections() error { } var sectionsToBeCreated []gcsql.Section - rows, err := m.db.QuerySQL(sectionsQuery) + rows, err := m.db.Query(nil, sectionsQuery) if err != nil { errEv.Err(err).Caller().Msg("Failed to query old database sections") return err @@ -126,7 +126,7 @@ func (m *Pre2021Migrator) MigrateBoards() error { } // get boards from old db - rows, err := m.db.QuerySQL(boardsQuery) + rows, err := m.db.Query(nil, boardsQuery) if err != nil { errEv.Err(err).Caller().Msg("Failed to query old database boards") return err @@ -162,7 +162,7 @@ func (m *Pre2021Migrator) MigrateBoards() error { Int("migratedBoardID", newBoard.ID). Msg("Board already exists in new db, updating values") // don't update other values in the array since they don't affect migrating threads or posts - if _, err = gcsql.ExecSQL(`UPDATE DBPREFIXboards + if _, err = gcsql.Exec(nil, `UPDATE DBPREFIXboards SET uri = ?, navbar_position = ?, title = ?, subtitle = ?, description = ?, max_file_size = ?, max_threads = ?, default_style = ?, locked = ?, anonymous_name = ?, force_anonymous = ?, autosage_after = ?, no_images_after = ?, max_message_length = ?, diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index bd4bfa56..3fa474fc 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -1,7 +1,6 @@ package pre2021 import ( - "context" "database/sql" "time" @@ -35,10 +34,10 @@ type migrationPost struct { func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *zerolog.Event) error { var err error - + opts := &gcsql.RequestOptions{Tx: tx} if post.oldParentID == 0 { // migrating post was a thread OP, create the row in the threads table - if post.ThreadID, err = gcsql.CreateThread(tx, post.boardID, false, post.stickied, post.autosage, false); err != nil { + if post.ThreadID, err = gcsql.CreateThread(opts, post.boardID, false, post.stickied, post.autosage, false); err != nil { errEv.Err(err).Caller(). Int("boardID", post.boardID). Msg("Failed to create thread") @@ -46,7 +45,7 @@ func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *ze } // insert thread top post - if err = post.InsertWithContext(context.Background(), tx, true, post.boardID, false, post.stickied, post.autosage, false); err != nil { + if err = post.Insert(true, post.boardID, false, post.stickied, post.autosage, false, opts); err != nil { errEv.Err(err).Caller(). Int("boardID", post.boardID). Int("threadID", post.ThreadID). @@ -54,7 +53,7 @@ func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *ze } if post.filename != "" { - if err = post.AttachFileTx(tx, &gcsql.Upload{ + if err = post.AttachFile(&gcsql.Upload{ PostID: post.ID, OriginalFilename: post.filenameOriginal, Filename: post.filename, @@ -64,7 +63,7 @@ func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *ze ThumbnailHeight: post.thumbH, Width: post.imageW, Height: post.imageH, - }); err != nil { + }, opts); err != nil { errEv.Err(err).Caller(). Int("oldPostID", post.oldID). Msg("Failed to attach upload to migrated post") @@ -85,7 +84,7 @@ func (m *Pre2021Migrator) MigratePosts() error { } defer tx.Rollback() - rows, err := m.db.QuerySQL(threadsQuery) + rows, err := m.db.Query(nil, threadsQuery) if err != nil { errEv.Err(err).Caller().Msg("Failed to get threads") return err @@ -126,7 +125,7 @@ func (m *Pre2021Migrator) MigratePosts() error { } // get and insert replies - replyRows, err := m.db.QuerySQL(postsQuery+" AND parentid = ?", thread.oldID) + replyRows, err := m.db.Query(nil, postsQuery+" AND parentid = ?", thread.oldID) if err != nil { errEv.Err(err).Caller(). Int("parentID", thread.oldID). @@ -156,7 +155,7 @@ func (m *Pre2021Migrator) MigratePosts() error { } if thread.locked { - if _, err = gcsql.ExecTxSQL(tx, "UPDATE DBPREFIXthreads SET locked = TRUE WHERE id = ?", thread.ThreadID); err != nil { + if _, err = gcsql.Exec(&gcsql.RequestOptions{Tx: tx}, "UPDATE DBPREFIXthreads SET locked = TRUE WHERE id = ?", thread.ThreadID); err != nil { errEv.Err(err).Caller(). Int("threadID", thread.ThreadID). Msg("Unable to re-lock migrated thread") diff --git a/pkg/gcsql/bans.go b/pkg/gcsql/bans.go index 2632fcd3..6bb1f086 100644 --- a/pkg/gcsql/bans.go +++ b/pkg/gcsql/bans.go @@ -25,38 +25,42 @@ type Ban interface { Deactivate(int) error } -func NewIPBanTx(tx *sql.Tx, ban *IPBan) error { +func NewIPBan(ban *IPBan, requestOpts ...*RequestOptions) error { const query = `INSERT INTO DBPREFIXip_ban (staff_id, board_id, banned_for_post_id, copy_post_text, is_thread_ban, is_active, range_start, range_end, appeal_at, expires_at, permanent, staff_note, message, can_appeal) VALUES(?, ?, ?, ?, ?, ?, PARAM_ATON, PARAM_ATON, ?, ?, ?, ?, ?, ?)` + opts := setupOptions(requestOpts...) + shouldCommit := opts.Tx == nil + var err error + if shouldCommit { + opts.Tx, err = BeginTx() + if err != nil { + return err + } + defer opts.Tx.Rollback() + } + if ban.ID > 0 { return ErrBanAlreadyInserted } - _, err := ExecTxSQL(tx, query, ban.StaffID, ban.BoardID, ban.BannedForPostID, ban.CopyPostText, + if _, err = Exec(opts, query, ban.StaffID, ban.BoardID, ban.BannedForPostID, ban.CopyPostText, ban.IsThreadBan, ban.IsActive, ban.RangeStart, ban.RangeEnd, ban.AppealAt, - ban.ExpiresAt, ban.Permanent, ban.StaffNote, ban.Message, ban.CanAppeal) + ban.ExpiresAt, ban.Permanent, ban.StaffNote, ban.Message, ban.CanAppeal, + ); err != nil { + return err + } + + ban.ID, err = getLatestID(opts, "DBPREFIXip_ban") if err != nil { return err } - - ban.ID, err = getLatestID("DBPREFIXip_ban", tx) - return err -} - -func NewIPBan(ban *IPBan) error { - tx, err := BeginTx() - if err != nil { - return err - } - defer tx.Rollback() - - if err = NewIPBanTx(tx, ban); err != nil { - return err + if shouldCommit { + return opts.Tx.Commit() } - return tx.Commit() + return nil } // CheckIPBan returns the latest active IP ban for the given IP, as well as any @@ -74,7 +78,7 @@ func CheckIPBan(ip string, boardID int) (*IPBan, error) { (expires_at > CURRENT_TIMESTAMP OR permanent) ORDER BY id DESC LIMIT 1` var ban IPBan - err := QueryRowSQL(query, []any{ip, ip, boardID}, []any{ + err := QueryRow(nil, query, []any{ip, ip, boardID}, []any{ &ban.ID, &ban.StaffID, &ban.BoardID, &ban.BannedForPostID, &ban.CopyPostText, &ban.IsThreadBan, &ban.IsActive, &ban.RangeStart, &ban.RangeEnd, &ban.IssuedAt, &ban.AppealAt, &ban.ExpiresAt, &ban.Permanent, &ban.StaffNote, &ban.Message, @@ -90,7 +94,7 @@ func CheckIPBan(ip string, boardID int) (*IPBan, error) { func GetIPBanByID(id int) (*IPBan, error) { const query = ipBanQueryBase + " WHERE id = ?" var ban IPBan - err := QueryRowSQL(query, []any{id}, []any{ + err := QueryRow(nil, query, []any{id}, []any{ &ban.ID, &ban.StaffID, &ban.BoardID, &ban.BannedForPostID, &ban.CopyPostText, &ban.IsThreadBan, &ban.IsActive, &ban.RangeStart, &ban.RangeEnd, &ban.IssuedAt, &ban.AppealAt, &ban.ExpiresAt, &ban.Permanent, &ban.StaffNote, &ban.Message, @@ -110,14 +114,15 @@ func GetIPBans(boardID int, limit int, onlyActive bool) ([]IPBan, error) { var rows *sql.Rows var err error if boardID > 0 { - rows, err = QuerySQL(query, boardID) + rows, err = Query(nil, query, boardID) } else { - rows, err = QuerySQL(query) + rows, err = Query(nil, query) } if err != nil { return nil, err } var bans []IPBan + defer rows.Close() for rows.Next() { var ban IPBan if err = rows.Scan( @@ -125,7 +130,6 @@ func GetIPBans(boardID int, limit int, onlyActive bool) ([]IPBan, error) { &ban.IsActive, &ban.RangeStart, &ban.RangeEnd, &ban.IssuedAt, &ban.AppealAt, &ban.ExpiresAt, &ban.Permanent, &ban.StaffNote, &ban.Message, &ban.CanAppeal, ); err != nil { - rows.Close() return nil, err } if onlyActive && !ban.IsActive { @@ -138,7 +142,7 @@ func GetIPBans(boardID int, limit int, onlyActive bool) ([]IPBan, error) { func (ipb *IPBan) Appeal(msg string) error { const query = `INSERT INTO DBPREFIXip_ban_appeals (ip_ban_id, appeal_text, is_denied) VALUES(?, ?, FALSE)` - _, err := ExecSQL(query, ipb.ID, msg) + _, err := Exec(nil, query, ipb.ID, msg) return err } @@ -163,10 +167,10 @@ func (ipb *IPBan) Deactivate(_ int) error { return err } defer tx.Rollback() - if _, err = ExecTxSQL(tx, deactivateQuery, ipb.ID); err != nil { + if _, err = Exec(&RequestOptions{Tx: tx}, deactivateQuery, ipb.ID); err != nil { return err } - if _, err = ExecTxSQL(tx, auditInsertQuery, ipb.ID); err != nil { + if _, err = Exec(&RequestOptions{Tx: tx}, auditInsertQuery, ipb.ID); err != nil { return err } return tx.Commit() diff --git a/pkg/gcsql/boards.go b/pkg/gcsql/boards.go index 744a29d2..87cbb281 100644 --- a/pkg/gcsql/boards.go +++ b/pkg/gcsql/boards.go @@ -459,7 +459,7 @@ func (board *Board) ModifyInDB() error { require_file = ?, enable_catalog = ? WHERE id = ?` - _, err := ExecSQL(query, + _, err := Exec(nil, query, board.SectionID, board.NavbarPosition, board.Title, board.Subtitle, board.Description, board.MaxFilesize, board.MaxThreads, board.DefaultStyle, board.Locked, board.AnonymousName, board.ForceAnonymous, board.AutosageAfter, board.NoImagesAfter, board.MaxMessageLength, diff --git a/pkg/gcsql/database.go b/pkg/gcsql/database.go index f9cdb51a..50d633f0 100644 --- a/pkg/gcsql/database.go +++ b/pkg/gcsql/database.go @@ -118,21 +118,16 @@ func (db *GCDB) PrepareContextSQL(ctx context.Context, query string, tx *sql.Tx) return stmt, sqlVersionError(err, db.driver, &prepared) } -/* -ExecSQL executes the given SQL statement with the given parameters -Example: - - var intVal int - var stringVal string - result, err := db.ExecSQL("INSERT INTO tablename (intval,stringval) VALUES(?,?)", intVal, stringVal) -*/ -func (db *GCDB) ExecSQL(query string, values ...any) (sql.Result, error) { - stmt, err := db.PrepareSQL(query, nil) +// Exec executes the given SQL statement with the given parameters, optionally with the given RequestOptions struct +// or a background context and transaction if nil +func (db *GCDB) Exec(opts *RequestOptions, query string, values ...any) (sql.Result, error) { + opts = setupOptions(opts) + stmt, err := db.PrepareContextSQL(opts.Context, query, opts.Tx) if err != nil { return nil, err } defer stmt.Close() - result, err := stmt.Exec(values...) + result, err := stmt.ExecContext(opts.Context, values...) if err != nil { return nil, err } @@ -140,7 +135,22 @@ func (db *GCDB) ExecSQL(query string, values ...any) (sql.Result, error) { } /* -ExecContextSQL executes the given SQL statement with the given context, optionally with the given transaction (if non-nil) +ExecSQL executes the given SQL statement with the given parameters. +Deprecated: Use Exec instead + +Example: + + var intVal int + var stringVal string + result, err := db.ExecSQL("INSERT INTO tablename (intval,stringval) VALUES(?,?)", intVal, stringVal) +*/ +func (db *GCDB) ExecSQL(query string, values ...any) (sql.Result, error) { + return db.Exec(nil, query, values...) +} + +/* +ExecContextSQL executes the given SQL statement with the given context, optionally with the given transaction (if non-nil). +Deprecated: Use Exec instead, with a RequestOptions struct for the context and transaction Example: @@ -152,21 +162,12 @@ Example: intVal, stringVal) */ func (db *GCDB) ExecContextSQL(ctx context.Context, tx *sql.Tx, sqlStr string, values ...any) (sql.Result, error) { - stmt, err := db.PrepareContextSQL(ctx, sqlStr, tx) - if err != nil { - return nil, err - } - defer stmt.Close() - - result, err := stmt.ExecContext(ctx, values...) - if err != nil { - return nil, err - } - return result, stmt.Close() + return db.Exec(&RequestOptions{Context: ctx, Tx: tx}, sqlStr, values...) } /* -ExecTxSQL executes the given SQL statemtnt, optionally with the given transaction (if non-nil) +ExecTxSQL executes the given SQL statemtnt, optionally with the given transaction (if non-nil). +Deprecated: Use Exec instead, with a RequestOptions struct for the transaction Example: @@ -179,16 +180,7 @@ Example: intVal, stringVal) */ func (db *GCDB) ExecTxSQL(tx *sql.Tx, query string, values ...any) (sql.Result, error) { - stmt, err := db.PrepareSQL(query, tx) - if err != nil { - return nil, err - } - defer stmt.Close() - res, err := stmt.Exec(values...) - if err != nil { - return res, err - } - return res, stmt.Close() + return db.Exec(&RequestOptions{Tx: tx}, query, values...) } /* @@ -209,9 +201,25 @@ func (db *GCDB) BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, erro return db.db.BeginTx(ctx, opts) } +// QueryRow gets a row from the db with the values in values[] and fills the respective pointers in out[], +// with an optional RequestOptions struct for the context and transaction +func (db *GCDB) QueryRow(opts *RequestOptions, query string, values []any, out []any) error { + opts = setupOptions(opts) + stmt, err := db.PrepareContextSQL(opts.Context, query, opts.Tx) + if err != nil { + return err + } + defer stmt.Close() + if err = stmt.QueryRowContext(opts.Context, values...).Scan(out...); err != nil { + return err + } + return stmt.Close() +} + /* -QueryRowSQL gets a row from the db with the values in values[] and fills the respective pointers in out[] -Automatically escapes the given values and caches the query +QueryRowSQL gets a row from the db with the values in values[] and fills the respective pointers in out[]. +Deprecated: Use QueryRow instead + Example: id := 32 @@ -222,12 +230,7 @@ Example: []any{&intVal, &stringVal}) */ func (db *GCDB) QueryRowSQL(query string, values, out []any) error { - stmt, err := db.PrepareSQL(query, nil) - if err != nil { - return err - } - defer stmt.Close() - return stmt.QueryRow(values...).Scan(out...) + return db.QueryRow(nil, query, values, out) } /* @@ -244,16 +247,7 @@ Example: []any{id}, []any{&name}) */ func (db *GCDB) QueryRowContextSQL(ctx context.Context, tx *sql.Tx, query string, values, out []any) error { - stmt, err := db.PrepareContextSQL(ctx, query, tx) - if err != nil { - return err - } - defer stmt.Close() - - if err = stmt.QueryRowContext(ctx, values...).Scan(out...); err != nil { - return err - } - return stmt.Close() + return db.QueryRow(&RequestOptions{Context: ctx, Tx: tx}, query, values, out) } /* @@ -273,21 +267,29 @@ Example: []any{&intVal, &stringVal}) */ func (db *GCDB) QueryRowTxSQL(tx *sql.Tx, query string, values, out []any) error { - stmt, err := db.PrepareSQL(query, tx) + return db.QueryRow(&RequestOptions{Tx: tx}, query, values, out) +} + +// Query sends the query to the database with the given options (or a background context if nil), and the given parameters +func (db *GCDB) Query(opts *RequestOptions, query string, a ...any) (*sql.Rows, error) { + opts = setupOptions(opts) + stmt, err := db.PrepareContextSQL(opts.Context, query, opts.Tx) if err != nil { - return err + return nil, err } defer stmt.Close() - if err = stmt.QueryRow(values...).Scan(out...); err != nil { - return err + rows, err := stmt.QueryContext(opts.Context, a...) + if err != nil { + return rows, err } - return stmt.Close() + + return rows, stmt.Close() } /* -QuerySQL gets all rows from the db with the values in values[] and fills the respective pointers in out[] -Automatically escapes the given values and caches the query +QuerySQL gets all rows from the db with the values in values[] and fills the respective pointers in out[]. +Deprecated: Use Query instead Example: rows, err := db.QuerySQL("SELECT * FROM table") @@ -301,17 +303,13 @@ Example: } */ func (db *GCDB) QuerySQL(query string, a ...any) (*sql.Rows, error) { - stmt, err := db.PrepareSQL(query, nil) - if err != nil { - return nil, err - } - defer stmt.Close() - return stmt.Query(a...) + return db.Query(nil, query, a...) } /* QueryContextSQL queries the database with a prepared statement and the given parameters, using the given context -for a deadline +for a deadline. +Deprecated: Use Query instead, with a RequestOptions struct for the context and transaction Example: @@ -320,22 +318,12 @@ Example: rows, err := db.QueryContextSQL(ctx, nil, "SELECT name from posts where NOT is_deleted") */ func (db *GCDB) QueryContextSQL(ctx context.Context, tx *sql.Tx, query string, a ...any) (*sql.Rows, error) { - stmt, err := db.PrepareContextSQL(ctx, query, tx) - if err != nil { - return nil, err - } - defer stmt.Close() - - rows, err := stmt.QueryContext(ctx, a...) - if err != nil { - return rows, err - } - return rows, stmt.Close() + return db.Query(&RequestOptions{Context: ctx, Tx: tx}, query, a...) } /* -QueryTxSQL gets all rows from the db with the values in values[] and fills the respective pointers in out[] -Automatically escapes the given values and caches the query +QueryTxSQL gets all rows from the db with the values in values[] and fills the respective pointers in out[]. +Deprecated: Use Query instead, with a RequestOptions struct for the transaction Example: tx, _ := db.Begin() @@ -350,16 +338,7 @@ Example: } */ func (db *GCDB) QueryTxSQL(tx *sql.Tx, query string, a ...any) (*sql.Rows, error) { - stmt, err := db.PrepareSQL(query, tx) - if err != nil { - return nil, err - } - defer stmt.Close() - rows, err := stmt.Query(a...) - if err != nil { - return nil, err - } - return rows, stmt.Close() + return db.Query(&RequestOptions{Tx: tx}, query, a...) } func setupDBConn(cfg *config.SQLConfig) (db *GCDB, err error) { @@ -404,6 +383,7 @@ func setupSqlTestConfig(dbDriver string, dbName string, dbPrefix string) *config } } +// SetupMockDB sets up a mock database connection for testing func SetupMockDB(driver string) (sqlmock.Sqlmock, error) { var err error gcdb, err = setupDBConn(setupSqlTestConfig(driver, "gochan", "")) diff --git a/pkg/gcsql/posts.go b/pkg/gcsql/posts.go index a8493d97..96479c1c 100644 --- a/pkg/gcsql/posts.go +++ b/pkg/gcsql/posts.go @@ -39,23 +39,24 @@ func GetPostFromID(id int, onlyNotDeleted bool) (*Post, error) { query += " AND is_deleted = FALSE" } post := new(Post) - err := QueryRowSQL(query, []any{id}, []any{ + err := QueryRow(nil, query, []any{id}, []any{ &post.ID, &post.ThreadID, &post.IsTopPost, &post.IP, &post.CreatedOn, &post.Name, &post.Tripcode, &post.IsRoleSignature, &post.Email, &post.Subject, &post.Message, &post.MessageRaw, &post.Password, &post.DeletedAt, &post.IsDeleted, &post.BannedMessage, &post.Flag, &post.Country, }) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, ErrPostDoesNotExist - + } else if err != nil { + return nil, err } - return post, err + return post, nil } func GetPostIP(postID int) (string, error) { sql := "SELECT IP_NTOA FROM DBPREFIXposts WHERE id = ?" var ip string - err := QueryRowSQL(sql, []any{postID}, []any{&ip}) + err := QueryRow(nil, sql, []any{postID}, []any{&ip}) return ip, err } @@ -68,7 +69,7 @@ func GetPostsFromIP(ip string, limit int, onlyNotDeleted bool) ([]Post, error) { } sql += " ORDER BY id DESC LIMIT ?" - rows, err := QuerySQL(sql, ip, limit) + rows, err := Query(nil, sql, ip, limit) if err != nil { return nil, err } @@ -109,7 +110,7 @@ func GetTopPostIDsInThreadIDs(threads ...any) (map[any]int, error) { } params := createArrayPlaceholder(threads) query := `SELECT id FROM DBPREFIXposts WHERE thread_id in ` + params + " AND is_top_post" - rows, err := QuerySQL(query, threads...) + rows, err := Query(nil, query, threads...) if err != nil { return nil, err } @@ -130,7 +131,7 @@ func GetTopPostIDsInThreadIDs(threads ...any) (map[any]int, error) { func GetThreadTopPost(threadID int) (*Post, error) { const query = selectPostsBaseSQL + "WHERE thread_id = ? AND is_top_post = TRUE LIMIT 1" post := new(Post) - err := QueryRowSQL(query, []any{threadID}, []any{ + err := QueryRow(nil, query, []any{threadID}, []any{ &post.ID, &post.ThreadID, &post.IsTopPost, &post.IP, &post.CreatedOn, &post.Name, &post.Tripcode, &post.IsRoleSignature, &post.Email, &post.Subject, &post.Message, &post.MessageRaw, &post.Password, &post.DeletedAt, &post.IsDeleted, @@ -180,19 +181,23 @@ func GetBoardTopPosts[B intOrStringConstraint](board B) ([]*Post, error) { func GetPostPassword(id int) (string, error) { const query = `SELECT password FROM DBPREFIXposts WHERE id = ?` var passwordChecksum string - err := QueryRowSQL(query, []any{id}, []any{&passwordChecksum}) + err := QueryRow(nil, query, []any{id}, []any{&passwordChecksum}) return passwordChecksum, err } // PermanentlyRemoveDeletedPosts removes all posts and files marked as deleted from the database -func PermanentlyRemoveDeletedPosts() error { +func PermanentlyRemoveDeletedPosts(opts ...*RequestOptions) error { const sql1 = `DELETE FROM DBPREFIXposts WHERE is_deleted` const sql2 = `DELETE FROM DBPREFIXthreads WHERE is_deleted` - _, err := ExecSQL(sql1) + var useOpts *RequestOptions + if len(opts) > 0 { + useOpts = opts[0] + } + _, err := Exec(useOpts, sql1) if err != nil { return err } - _, err = ExecSQL(sql2) + _, err = Exec(useOpts, sql2) return err } @@ -201,7 +206,7 @@ func PermanentlyRemoveDeletedPosts() error { func SinceLastPost(postIP string) (int, error) { const query = `SELECT COALESCE(MAX(created_on), '1970-01-01 00:00:00') FROM DBPREFIXposts WHERE ip = ?` var whenStr string - err := QueryRowSQL(query, []any{postIP}, []any{&whenStr}) + err := QueryRow(nil, query, []any{postIP}, []any{&whenStr}) if err != nil { return -1, err } @@ -219,7 +224,7 @@ func SinceLastThread(postIP string) (int, error) { const query = `SELECT COALESCE(MAX(created_on), '1970-01-01 00:00:00') FROM DBPREFIXposts WHERE ip = ? AND is_top_post` var whenStr string - err := QueryRowSQL(query, []any{postIP}, []any{&whenStr}) + err := QueryRow(nil, query, []any{postIP}, []any{&whenStr}) if err != nil { return -1, err } @@ -233,7 +238,7 @@ func SinceLastThread(postIP string) (int, error) { // UpdateContents updates the email, subject, and message text of the post func (p *Post) UpdateContents(email string, subject string, message template.HTML, messageRaw string) error { const sqlUpdate = `UPDATE DBPREFIXposts SET email = ?, subject = ?, message = ?, message_raw = ? WHERE ID = ?` - _, err := ExecSQL(sqlUpdate, email, subject, message, messageRaw, p.ID) + _, err := Exec(nil, sqlUpdate, email, subject, message, messageRaw, p.ID) if err != nil { return err } @@ -244,27 +249,27 @@ func (p *Post) UpdateContents(email string, subject string, message template.HTM return nil } -func (p *Post) GetBoardID() (int, error) { +func (p *Post) GetBoardID(opts ...*RequestOptions) (int, error) { const query = `SELECT board_id FROM DBPREFIXthreads where id = ?` var boardID int - err := QueryRowSQL(query, []any{p.ThreadID}, []any{&boardID}) + err := QueryRow(setupOptions(opts...), query, []any{p.ThreadID}, []any{&boardID}) if errors.Is(err, sql.ErrNoRows) { err = ErrBoardDoesNotExist } return boardID, err } -func (p *Post) GetBoardDir() (string, error) { +func (p *Post) GetBoardDir(opts ...*RequestOptions) (string, error) { const query = "SELECT dir FROM DBPREFIXboards" + boardFromPostIdSuffixSQL var dir string - err := QueryRowSQL(query, []any{p.ID}, []any{&dir}) + err := QueryRow(setupOptions(opts...), query, []any{p.ID}, []any{&dir}) return dir, err } -func (p *Post) GetBoard() (*Board, error) { +func (p *Post) GetBoard(opts ...*RequestOptions) (*Board, error) { const query = selectBoardsBaseSQL + boardFromPostIdSuffixSQL board := new(Board) - err := QueryRowSQL(query, []any{p.ID}, []any{ + err := QueryRow(setupOptions(opts...), query, []any{p.ID}, []any{ &board.ID, &board.SectionID, &board.URI, &board.Dir, &board.NavbarPosition, &board.Title, &board.Subtitle, &board.Description, &board.MaxFilesize, &board.MaxThreads, &board.DefaultStyle, &board.Locked, &board.CreatedAt, &board.AnonymousName, &board.ForceAnonymous, &board.AutosageAfter, &board.NoImagesAfter, @@ -284,13 +289,13 @@ func (p *Post) ChangeBoardID(newBoardID int) error { } // TopPostID returns the OP post ID of the thread that p is in -func (p *Post) TopPostID() (int, error) { +func (p *Post) TopPostID(opts ...*RequestOptions) (int, error) { if p.IsTopPost { return p.ID, nil } const query = `SELECT id FROM DBPREFIXposts WHERE thread_id = ? and is_top_post = TRUE ORDER BY id ASC LIMIT 1` var topPostID int - err := QueryRowSQL(query, []any{p.ThreadID}, []any{&topPostID}) + err := QueryRow(setupOptions(opts...), query, []any{p.ThreadID}, []any{&topPostID}) return topPostID, err } @@ -306,13 +311,13 @@ func (p *Post) GetTopPost() (*Post, error) { // GetPostUpload returns the upload info associated with the file as well as any errors encountered. // If the file has no uploads, then *Upload is nil. If the file was removed from the post, then Filename // and OriginalFilename = "deleted" -func (p *Post) GetUpload() (*Upload, error) { +func (p *Post) GetUpload(opts ...*RequestOptions) (*Upload, error) { const query = `SELECT id, post_id, file_order, original_filename, filename, checksum, file_size, is_spoilered, thumbnail_width, thumbnail_height, width, height FROM DBPREFIXfiles WHERE post_id = ?` upload := new(Upload) - err := QueryRowSQL(query, []any{p.ID}, []any{ + err := QueryRow(setupOptions(opts...), query, []any{p.ID}, []any{ &upload.ID, &upload.PostID, &upload.FileOrder, &upload.OriginalFilename, &upload.Filename, &upload.Checksum, &upload.FileSize, &upload.IsSpoilered, &upload.ThumbnailWidth, &upload.ThumbnailHeight, &upload.Width, &upload.Height, }) @@ -325,7 +330,7 @@ func (p *Post) GetUpload() (*Upload, error) { // UnlinkUploads disassociates the post with any uploads in DBPREFIXfiles // that may have been uploaded with it, optionally leaving behind a "File Deleted" // frame where the thumbnail appeared -func (p *Post) UnlinkUploads(leaveDeletedBox bool) error { +func (p *Post) UnlinkUploads(leaveDeletedBox bool, requestOpts ...*RequestOptions) error { var sqlStr string if leaveDeletedBox { // leave a "File Deleted" box @@ -333,7 +338,7 @@ func (p *Post) UnlinkUploads(leaveDeletedBox bool) error { } else { sqlStr = `DELETE FROM DBPREFIXfiles WHERE post_id = ?` } - _, err := ExecSQL(sqlStr, p.ID) + _, err := Exec(setupOptions(requestOpts...), sqlStr, p.ID) return err } @@ -348,17 +353,24 @@ func (p *Post) InCyclicThread() (bool, error) { } // Delete sets the post as deleted and sets the deleted_at timestamp to the current time -func (p *Post) Delete() error { - ctx, cancel := context.WithTimeout(context.Background(), gcdb.defaultTimeout) - defer cancel() - tx, err := BeginContextTx(ctx) - if err != nil { - return err +func (p *Post) Delete(requestOptions ...*RequestOptions) error { + shouldCommit := len(requestOptions) == 0 + opts := setupOptions(requestOptions...) + if opts.Context == context.Background() { + opts.Context, opts.Cancel = context.WithTimeout(context.Background(), gcdb.defaultTimeout) + defer opts.Cancel() + } + var err error + if opts.Tx == nil { + opts.Tx, err = BeginContextTx(opts.Context) + if err != nil { + return err + } + defer opts.Tx.Rollback() } - defer tx.Rollback() var rowCount int - err = QueryRowContextSQL(ctx, tx, "SELECT COUNT(*) FROM DBPREFIXposts WHERE id = ?", []any{p.ID}, []any{&rowCount}) + err = QueryRow(opts, "SELECT COUNT(*) FROM DBPREFIXposts WHERE id = ?", []any{p.ID}, []any{&rowCount}) if errors.Is(err, sql.ErrNoRows) { err = ErrPostDoesNotExist } @@ -367,16 +379,33 @@ func (p *Post) Delete() error { } if p.IsTopPost { - return deleteThread(ctx, tx, p.ThreadID) + return deleteThread(opts, p.ThreadID) } - if _, err = ExecContextSQL(ctx, tx, "UPDATE DBPREFIXposts SET is_deleted = TRUE, deleted_at = CURRENT_TIMESTAMP WHERE id = ?", p.ID); err != nil { + if _, err = Exec(opts, "UPDATE DBPREFIXposts SET is_deleted = TRUE, deleted_at = CURRENT_TIMESTAMP WHERE id = ?", p.ID); err != nil { return err } - return tx.Commit() + if shouldCommit { + return opts.Tx.Commit() + } + return nil } -// InsertWithContext inserts the post into the database with the given context and transaction -func (p *Post) InsertWithContext(ctx context.Context, tx *sql.Tx, bumpThread bool, boardID int, locked bool, stickied bool, anchored bool, cyclical bool) error { +// Insert inserts the post into the database with the optional given options +func (p *Post) Insert(bumpThread bool, boardID int, locked bool, stickied bool, anchored bool, cyclical bool, requestOptions ...*RequestOptions) error { + opts := setupOptions(requestOptions...) + if len(requestOptions) == 0 { + opts.Context, opts.Cancel = context.WithTimeout(context.Background(), gcdb.defaultTimeout) + defer opts.Cancel() + } + var err error + if opts.Tx == nil { + opts.Tx, err = BeginContextTx(opts.Context) + if err != nil { + return err + } + defer opts.Tx.Rollback() + } + if p.ID > 0 { // already inserted return ErrorPostAlreadySent @@ -387,19 +416,18 @@ func (p *Post) InsertWithContext(ctx context.Context, tx *sql.Tx, bumpThread boo VALUES(?,?,PARAM_ATON,CURRENT_TIMESTAMP,?,?,?,?,?,?,?,?,?,?)` bumpSQL := `UPDATE DBPREFIXthreads SET last_bump = CURRENT_TIMESTAMP WHERE id = ?` - var err error if p.ThreadID == 0 { // thread doesn't exist yet, this is a new post p.IsTopPost = true var threadID int - threadID, err = CreateThread(tx, boardID, locked, stickied, anchored, cyclical) + threadID, err = CreateThread(opts, boardID, locked, stickied, anchored, cyclical) if err != nil { return err } p.ThreadID = threadID } else { var threadIsLocked bool - if err = QueryRowTxSQL(tx, "SELECT locked FROM DBPREFIXthreads WHERE id = ?", + if err = QueryRow(opts, "SELECT locked FROM DBPREFIXthreads WHERE id = ?", []any{p.ThreadID}, []any{&threadIsLocked}); err != nil { return err } @@ -408,40 +436,26 @@ func (p *Post) InsertWithContext(ctx context.Context, tx *sql.Tx, bumpThread boo } } - if _, err = ExecContextSQL(ctx, tx, insertSQL, + if _, err = Exec(opts, insertSQL, p.ThreadID, p.IsTopPost, p.IP, p.Name, p.Tripcode, p.IsRoleSignature, p.Email, p.Subject, p.Message, p.MessageRaw, p.Password, p.Flag, p.Country, ); err != nil { return err } - if p.ID, err = getLatestID("DBPREFIXposts", tx); err != nil { + if p.ID, err = getLatestID(opts, "DBPREFIXposts"); err != nil { return err } if bumpThread { - if _, err = ExecContextSQL(ctx, tx, bumpSQL, p.ThreadID); err != nil { + if _, err = Exec(opts, bumpSQL, p.ThreadID); err != nil { return err } } + if len(requestOptions) == 0 { + return opts.Tx.Commit() + } return nil } -func (p *Post) Insert(bumpThread bool, boardID int, locked bool, stickied bool, anchored bool, cyclical bool) error { - ctx, cancel := context.WithTimeout(context.Background(), gcdb.defaultTimeout) - defer cancel() - - tx, err := BeginContextTx(ctx) - if err != nil { - return err - } - defer tx.Rollback() - - if err = p.InsertWithContext(ctx, tx, bumpThread, boardID, locked, stickied, anchored, cyclical); err != nil { - return err - } - - return tx.Commit() -} - // CyclicThreadPost represents a post that should be deleted in a cyclic thread type CyclicThreadPost struct { PostID int // sql: post_id @@ -515,7 +529,7 @@ func (p *Post) WebPath() string { webRoot := config.GetSystemCriticalConfig().WebRoot const query = "SELECT op_id, dir FROM DBPREFIXv_top_post_board_dir WHERE id = ?" - err := QueryRowSQL(query, []any{p.ID}, []any{&p.opID, &p.boardDir}) + err := QueryRow(nil, query, []any{p.ID}, []any{&p.opID, &p.boardDir}) if err != nil { return webRoot } diff --git a/pkg/gcsql/preload.go b/pkg/gcsql/preload.go index 401851a4..3312577b 100644 --- a/pkg/gcsql/preload.go +++ b/pkg/gcsql/preload.go @@ -39,7 +39,7 @@ func PreloadModule(l *lua.LState) int { }) } - rows, err := QuerySQL(queryStr, queryArgs...) + rows, err := Query(nil, queryStr, queryArgs...) l.Push(luar.New(l, rows)) l.Push(luar.New(l, err)) @@ -57,7 +57,7 @@ func PreloadModule(l *lua.LState) int { execArgs = append(execArgs, arg) }) } - result, err := ExecSQL(execStr) + result, err := Exec(nil, execStr) l.Push(luar.New(l, result)) l.Push(luar.New(l, err)) diff --git a/pkg/gcsql/sections.go b/pkg/gcsql/sections.go index 1a82a74b..ccfa9a0e 100644 --- a/pkg/gcsql/sections.go +++ b/pkg/gcsql/sections.go @@ -96,7 +96,7 @@ func GetSectionFromName(name string) (*Section, error) { // DeleteSection deletes a section from the database and resets the AllSections array func DeleteSection(id int) error { const query = `DELETE FROM DBPREFIXsections WHERE id = ?` - _, err := ExecSQL(query, id) + _, err := Exec(nil, query, id) if err != nil { return err } @@ -105,38 +105,47 @@ func DeleteSection(id int) error { // NewSection creates a new board section in the database and returns a *Section struct pointer. // If position < 0, it will use the ID -func NewSection(name string, abbreviation string, hidden bool, position int) (*Section, error) { +func NewSection(name string, abbreviation string, hidden bool, position int, requestOpts ...*RequestOptions) (*Section, error) { const sqlINSERT = `INSERT INTO DBPREFIXsections (name, abbreviation, hidden, position) VALUES (?,?,?,?)` const sqlPosition = `SELECT COALESCE(MAX(position) + 1, 1) FROM DBPREFIXsections` - - tx, err := BeginTx() - if err != nil { - return nil, err + var opts *RequestOptions + var err error + shouldCommit := len(requestOpts) == 0 + if shouldCommit { + opts = &RequestOptions{} + opts.Tx, err = BeginTx() + if err != nil { + return nil, err + } + opts.Context, opts.Cancel = context.WithTimeout(context.Background(), gcdb.defaultTimeout) + defer func() { + opts.Cancel() + opts.Tx.Rollback() + }() + } else { + opts = requestOpts[0] } - ctx, cancel := context.WithTimeout(context.Background(), gcdb.defaultTimeout) - defer func() { - cancel() - tx.Rollback() - }() if position < 0 { // position not specified - err = QueryRowContextSQL(ctx, tx, sqlPosition, nil, []any{&position}) + err = QueryRow(opts, sqlPosition, nil, []any{&position}) if errors.Is(err, sql.ErrNoRows) { position = 1 } else if err != nil { return nil, err } } - if _, err = ExecContextSQL(ctx, tx, sqlINSERT, name, abbreviation, hidden, position); err != nil { + if _, err = Exec(opts, sqlINSERT, name, abbreviation, hidden, position); err != nil { return nil, err } - id, err := getLatestID("DBPREFIXsections", tx) + id, err := getLatestID(opts, "DBPREFIXsections") if err != nil { return nil, err } - if err = tx.Commit(); err != nil { - return nil, err + if shouldCommit { + if err = opts.Tx.Commit(); err != nil { + return nil, err + } } return &Section{ ID: id, @@ -147,9 +156,15 @@ func NewSection(name string, abbreviation string, hidden bool, position int) (*S }, nil } -func (s *Section) UpdateValues() error { +func (s *Section) UpdateValues(requestOpts ...*RequestOptions) error { + opts := setupOptions(requestOpts...) + if opts.Context == context.Background() { + opts.Context, opts.Cancel = context.WithTimeout(context.Background(), gcdb.defaultTimeout) + defer opts.Cancel() + } + var count int - err := QueryRowTimeoutSQL(nil, `SELECT COUNT(*) FROM DBPREFIXsections WHERE id = ?`, []any{s.ID}, []any{&count}) + err := QueryRow(opts, `SELECT COUNT(*) FROM DBPREFIXsections WHERE id = ?`, []any{s.ID}, []any{&count}) if errors.Is(err, sql.ErrNoRows) { return ErrSectionDoesNotExist } @@ -157,6 +172,6 @@ func (s *Section) UpdateValues() error { return err } const query = `UPDATE DBPREFIXsections set name = ?, abbreviation = ?, position = ?, hidden = ? WHERE id = ?` - _, err = ExecTimeoutSQL(nil, query, s.Name, s.Abbreviation, s.Position, s.Hidden, s.ID) + _, err = Exec(opts, query, s.Name, s.Abbreviation, s.Position, s.Hidden, s.ID) return err } diff --git a/pkg/gcsql/setup_test.go b/pkg/gcsql/setup_test.go index 2ecb059c..9bef7237 100644 --- a/pkg/gcsql/setup_test.go +++ b/pkg/gcsql/setup_test.go @@ -171,7 +171,7 @@ func setupAndProvisionMockDB(t *testing.T, mock sqlmock.Sqlmock, dbType string, ExpectQuery().WithArgs("test"). WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) - _, err := ExecSQL("CREATE DATABASE gochan") + _, err := Exec(nil, "CREATE DATABASE gochan") if err != nil { return err } diff --git a/pkg/gcsql/threads.go b/pkg/gcsql/threads.go index ae02a980..99f65446 100644 --- a/pkg/gcsql/threads.go +++ b/pkg/gcsql/threads.go @@ -1,7 +1,6 @@ package gcsql import ( - "context" "database/sql" "errors" "fmt" @@ -21,27 +20,27 @@ var ( ) // CreateThread creates a new thread in the database with the given board ID and statuses -func CreateThread(tx *sql.Tx, boardID int, locked bool, stickied bool, anchored bool, cyclic bool) (threadID int, err error) { +func CreateThread(requestOptions *RequestOptions, boardID int, locked bool, stickied bool, anchored bool, cyclic bool) (threadID int, err error) { const lockedQuery = `SELECT locked FROM DBPREFIXboards WHERE id = ?` const insertQuery = `INSERT INTO DBPREFIXthreads (board_id, locked, stickied, anchored, cyclical) VALUES (?,?,?,?,?)` var boardIsLocked bool - if err = QueryRowTxSQL(tx, lockedQuery, []any{boardID}, []any{&boardIsLocked}); err != nil { + if err = QueryRow(requestOptions, lockedQuery, []any{boardID}, []any{&boardIsLocked}); err != nil { return 0, err } if boardIsLocked { return 0, ErrBoardIsLocked } - if _, err = ExecTxSQL(tx, insertQuery, boardID, locked, stickied, anchored, cyclic); err != nil { + if _, err = Exec(requestOptions, insertQuery, boardID, locked, stickied, anchored, cyclic); err != nil { return 0, err } - return threadID, QueryRowTxSQL(tx, "SELECT MAX(id) FROM DBPREFIXthreads", nil, []any{&threadID}) + return threadID, QueryRow(requestOptions, "SELECT MAX(id) FROM DBPREFIXthreads", nil, []any{&threadID}) } // GetThread returns a a thread object from the database, given its ID func GetThread(threadID int) (*Thread, error) { const query = selectThreadsBaseSQL + `WHERE id = ?` thread := new(Thread) - err := QueryRowSQL(query, []any{threadID}, []any{ + err := QueryRow(nil, query, []any{threadID}, []any{ &thread.ID, &thread.BoardID, &thread.Locked, &thread.Stickied, &thread.Anchored, &thread.Cyclic, &thread.LastBump, &thread.DeletedAt, &thread.IsDeleted, }) @@ -52,7 +51,7 @@ func GetThread(threadID int) (*Thread, error) { func GetPostThread(opID int) (*Thread, error) { const query = selectThreadsBaseSQL + `WHERE id = (SELECT thread_id FROM DBPREFIXposts WHERE id = ? LIMIT 1)` thread := new(Thread) - err := QueryRowSQL(query, []any{opID}, []any{ + err := QueryRow(nil, query, []any{opID}, []any{ &thread.ID, &thread.BoardID, &thread.Locked, &thread.Stickied, &thread.Anchored, &thread.Cyclic, &thread.LastBump, &thread.DeletedAt, &thread.IsDeleted, }) @@ -66,7 +65,7 @@ func GetPostThread(opID int) (*Thread, error) { func GetTopPostThreadID(opID int) (int, error) { const query = `SELECT thread_id FROM DBPREFIXposts WHERE id = ? and is_top_post` var threadID int - err := QueryRowSQL(query, []any{opID}, []any{&threadID}) + err := QueryRow(nil, query, []any{opID}, []any{&threadID}) if err == sql.ErrNoRows { err = ErrThreadDoesNotExist } @@ -81,7 +80,7 @@ func GetThreadsWithBoardID(boardID int, onlyNotDeleted bool) ([]Thread, error) { if onlyNotDeleted { query += " AND is_deleted = FALSE" } - rows, err := QuerySQL(query, boardID) + rows, err := Query(nil, query, boardID) if err != nil { return nil, err } @@ -104,7 +103,7 @@ func GetThreadReplyCountFromOP(opID int) (int, error) { const query = `SELECT COUNT(*) FROM DBPREFIXposts WHERE thread_id = ( SELECT thread_id FROM DBPREFIXposts WHERE id = ?) AND is_deleted = FALSE AND is_top_post = FALSE` var num int - err := QueryRowSQL(query, []any{opID}, []any{&num}) + err := QueryRow(nil, query, []any{opID}, []any{&num}) return num, err } @@ -113,7 +112,7 @@ func ChangeThreadBoardID(threadID int, newBoardID int) error { if !DoesBoardExistByID(newBoardID) { return ErrBoardDoesNotExist } - _, err := ExecSQL(`UPDATE DBPREFIXthreads SET board_id = ? WHERE id = ?`, newBoardID, threadID) + _, err := Exec(nil, "UPDATE DBPREFIXthreads SET board_id = ? WHERE id = ?", newBoardID, threadID) return err } @@ -135,15 +134,15 @@ func (t *Thread) GetReplyFileCount() (int, error) { const query = `SELECT COUNT(filename) FROM DBPREFIXfiles WHERE post_id IN ( SELECT id FROM DBPREFIXposts WHERE thread_id = ? AND is_deleted = FALSE)` var fileCount int - err := QueryRowSQL(query, []any{t.ID}, []any{&fileCount}) + err := QueryRow(nil, query, []any{t.ID}, []any{&fileCount}) return fileCount, err } // GetReplyCount returns the number of posts in the thread, not including the top post or any deleted posts func (t *Thread) GetReplyCount() (int, error) { - const query = `SELECT COUNT(*) FROM DBPREFIXposts WHERE thread_id = ? AND is_top_post = FALSE AND is_deleted = FALSE` + const query = "SELECT COUNT(*) FROM DBPREFIXposts WHERE thread_id = ? AND is_top_post = FALSE AND is_deleted = FALSE" var numReplies int - err := QueryRowSQL(query, []any{t.ID}, []any{&numReplies}) + err := QueryRow(nil, query, []any{t.ID}, []any{&numReplies}) return numReplies, err } @@ -161,7 +160,7 @@ func (t *Thread) GetPosts(repliesOnly bool, boardPage bool, limit int) ([]Post, query += " LIMIT " + strconv.Itoa(limit) } - rows, err := QuerySQL(query, t.ID) + rows, err := Query(nil, query, t.ID) if err != nil { return nil, err } @@ -185,7 +184,7 @@ func (t *Thread) GetPosts(repliesOnly bool, boardPage bool, limit int) ([]Post, func (t *Thread) GetUploads() ([]Upload, error) { const query = selectFilesBaseSQL + ` WHERE post_id IN ( SELECT id FROM DBPREFIXposts WHERE thread_id = ? and is_deleted = FALSE) AND filename != 'deleted'` - rows, err := QuerySQL(query, t.ID) + rows, err := Query(nil, query, t.ID) if err != nil { return nil, err } @@ -223,18 +222,18 @@ func (t *Thread) UpdateAttribute(attribute string, value bool) error { return fmt.Errorf("invalid thread attribute %q", attribute) } updateSQL += attribute + " = ? WHERE id = ?" - _, err := ExecSQL(updateSQL, value, t.ID) + _, err := Exec(nil, updateSQL, value, t.ID) return err } // deleteThread updates the thread and sets it as deleted, as well as the posts where thread_id = threadID -func deleteThread(ctx context.Context, tx *sql.Tx, threadID int) error { +func deleteThread(opts *RequestOptions, threadID int) error { const checkPostExistsSQL = `SELECT COUNT(*) FROM DBPREFIXposts WHERE thread_id = ?` const deletePostsSQL = `UPDATE DBPREFIXposts SET is_deleted = TRUE, deleted_at = CURRENT_TIMESTAMP WHERE thread_id = ?` const deleteThreadSQL = `UPDATE DBPREFIXthreads SET is_deleted = TRUE, deleted_at = CURRENT_TIMESTAMP WHERE id = ?` var rowCount int - err := QueryRowContextSQL(ctx, tx, checkPostExistsSQL, []any{threadID}, []any{&rowCount}) + err := QueryRow(opts, checkPostExistsSQL, []any{threadID}, []any{&rowCount}) if err != nil { return err } @@ -242,10 +241,10 @@ func deleteThread(ctx context.Context, tx *sql.Tx, threadID int) error { return ErrThreadDoesNotExist } - _, err = ExecContextSQL(ctx, tx, deletePostsSQL, threadID) + _, err = Exec(opts, deletePostsSQL, threadID) if err != nil { return err } - _, err = ExecContextSQL(ctx, tx, deleteThreadSQL, threadID) + _, err = Exec(opts, deleteThreadSQL, threadID) return err } diff --git a/pkg/gcsql/uploads.go b/pkg/gcsql/uploads.go index c6c8202b..9629889d 100644 --- a/pkg/gcsql/uploads.go +++ b/pkg/gcsql/uploads.go @@ -1,7 +1,6 @@ package gcsql import ( - "context" "database/sql" "errors" "fmt" @@ -25,7 +24,7 @@ func GetThreadFiles(post *Post) ([]Upload, error) { query := selectFilesBaseSQL + `WHERE post_id IN ( SELECT id FROM DBPREFIXposts WHERE thread_id = ( SELECT thread_id FROM DBPREFIXposts WHERE id = ?)) AND filename != 'deleted'` - rows, err := QuerySQL(query, post.ID) + rows, err := Query(nil, query, post.ID) if err != nil { return nil, err } @@ -45,17 +44,28 @@ func GetThreadFiles(post *Post) ([]Upload, error) { } // NextFileOrder gets what would be the next file_order value (not particularly useful until multi-file posting is implemented) -func (p *Post) NextFileOrder(ctx context.Context, tx *sql.Tx) (int, error) { +func (p *Post) NextFileOrder(requestOpts ...*RequestOptions) (int, error) { + opts := setupOptions(requestOpts...) const query = `SELECT COALESCE(MAX(file_order) + 1, 0) FROM DBPREFIXfiles WHERE post_id = ?` var next int - err := QueryRowContextSQL(ctx, tx, query, []any{p.ID}, []any{&next}) + err := QueryRow(opts, query, []any{p.ID}, []any{&next}) return next, err } -func (p *Post) AttachFileTx(tx *sql.Tx, upload *Upload) error { +func (p *Post) AttachFile(upload *Upload, requestOpts ...*RequestOptions) error { if upload == nil { return nil // no upload to attach, so no error } + opts := setupOptions(requestOpts...) + shouldCommit := opts.Tx == nil + var err error + if shouldCommit { + opts.Tx, err = BeginTx() + if err != nil { + return err + } + defer opts.Tx.Rollback() + } _, err, recovered := events.TriggerEvent("incoming-upload", upload) if recovered { @@ -72,35 +82,30 @@ func (p *Post) AttachFileTx(tx *sql.Tx, upload *Upload) error { if upload.ID > 0 { return ErrAlreadyAttached } - if upload.FileOrder < 1 { - upload.FileOrder, err = p.NextFileOrder(context.Background(), tx) + upload.FileOrder, err = p.NextFileOrder(opts) if err != nil { return err } } upload.PostID = p.ID - if _, err = ExecTxSQL(tx, insertSQL, + if _, err = Exec(opts, insertSQL, &upload.PostID, &upload.FileOrder, &upload.OriginalFilename, &upload.Filename, &upload.Checksum, &upload.FileSize, &upload.IsSpoilered, &upload.ThumbnailWidth, &upload.ThumbnailHeight, &upload.Width, &upload.Height, ); err != nil { return err } - upload.ID, err = getLatestID("DBPREFIXfiles", tx) - return err -} - -func (p *Post) AttachFile(upload *Upload) error { - tx, err := BeginTx() + upload.ID, err = getLatestID(opts, "DBPREFIXfiles") if err != nil { return err } - defer tx.Rollback() - if err = p.AttachFileTx(tx, upload); err != nil { - return err + if shouldCommit { + if err = opts.Tx.Commit(); err != nil { + return err + } } - return tx.Commit() + return nil } // GetUploadFilenameAndBoard returns the filename (or an empty string) and @@ -112,7 +117,7 @@ func GetUploadFilenameAndBoard(postID int) (string, string, error) { JOIN DBPREFIXboards ON DBPREFIXboards.id = board_id WHERE DBPREFIXposts.id = ?` var filename, dir string - err := QueryRowSQL(query, []any{postID}, []any{&filename, &dir}) + err := QueryRow(nil, query, []any{postID}, []any{&filename, &dir}) if errors.Is(err, sql.ErrNoRows) { return "", "", nil } else if err != nil { diff --git a/pkg/gcsql/util.go b/pkg/gcsql/util.go index 872b286d..a82ad968 100644 --- a/pkg/gcsql/util.go +++ b/pkg/gcsql/util.go @@ -51,6 +51,39 @@ type intOrStringConstraint interface { int | string } +// RequestOptions is used to pass an optional context, transaction, and any other things to the various SQL functions +// in a future-proof way +type RequestOptions struct { + Context context.Context + Tx *sql.Tx + Cancel context.CancelFunc +} + +func setupOptions(opts ...*RequestOptions) *RequestOptions { + if len(opts) == 0 || opts[0] == nil { + return &RequestOptions{Context: context.Background()} + } + if opts[0].Context == nil { + opts[0].Context = context.Background() + } + return opts[0] +} + +// Query is a wrapper for QueryContextSQL that uses the given options, or defaults to a background context if nil +func Query(opts *RequestOptions, query string, a ...any) (*sql.Rows, error) { + return gcdb.Query(opts, query, a...) +} + +// QueryRow is a wrapper for QueryRowContextSQL that uses the given options, or defaults to a background context if nil +func QueryRow(opts *RequestOptions, query string, values, out []any) error { + return gcdb.QueryRow(opts, query, values, out) +} + +// Exec is a wrapper for ExecContextSQL that uses the given options, or defaults to a background context if nil +func Exec(opts *RequestOptions, query string, values ...any) (sql.Result, error) { + return gcdb.Exec(opts, query, values...) +} + // BeginTx begins a new transaction for the gochan database. It uses a background context func BeginTx() (*sql.Tx, error) { return BeginContextTx(context.Background()) @@ -111,7 +144,7 @@ func SetupSQLString(query string, dbConn *GCDB) (string, error) { return prepared, err } -// Close closes the connection to the SQL database +// Close closes the connection to the SQL database if it is open func Close() error { if gcdb != nil { return gcdb.Close() @@ -121,6 +154,7 @@ func Close() error { /* ExecSQL executes the given SQL statement with the given parameters + Example: var intVal int @@ -136,8 +170,8 @@ func ExecSQL(query string, values ...any) (sql.Result, error) { } /* -ExecContextSQL executes the given SQL statement with the given context, optionally with the given transaction (if non-nil) - +ExecContextSQL executes the given SQL statement with the given context, optionally with the given transaction (if non-nil). +Deprecated: Use Exec instead Example: ctx, cancel := context.WithTimeout(context.Background(), time.Duration(sqlCfg.DBTimeoutSeconds) * time.Second) @@ -154,6 +188,7 @@ func ExecContextSQL(ctx context.Context, tx *sql.Tx, sqlStr string, values ...an return gcdb.ExecContextSQL(ctx, tx, sqlStr, values...) } +// ExecTimeoutSQL is a helper function for executing a SQL statement with the configured timeout in seconds func ExecTimeoutSQL(tx *sql.Tx, sqlStr string, values ...any) (sql.Result, error) { ctx, cancel := context.WithTimeout(context.Background(), gcdb.defaultTimeout) defer cancel() @@ -162,7 +197,9 @@ func ExecTimeoutSQL(tx *sql.Tx, sqlStr string, values ...any) (sql.Result, error } /* -ExecTxSQL automatically escapes the given values and caches the statement +ExecTxSQL executes the given SQL statement with the given transaction and parameters. +Deprecated: Use Exec instead with a transaction in the RequestOptions + Example: tx, err := BeginTx() @@ -190,8 +227,8 @@ func ExecTxSQL(tx *sql.Tx, sqlStr string, values ...any) (sql.Result, error) { } /* -QueryRowSQL gets a row from the db with the values in values[] and fills the respective pointers in out[] -Automatically escapes the given values and caches the query +QueryRowSQL gets a row from the db with the values in values[] and fills the respective pointers in out[]. +Deprecated: Use QueryRow instead Example: @@ -211,7 +248,8 @@ func QueryRowSQL(query string, values, out []any) error { /* QueryRowContextSQL gets a row from the database with the values in values[] and fills the respective pointers in out[] -using the given context as a deadline, and the given transaction (if non-nil) +using the given context as a deadline, and the given transaction (if non-nil). +Deprecated: Use QueryRow instead with an optional context and/or tx in the RequestOptions Example: @@ -239,8 +277,8 @@ func QueryRowTimeoutSQL(tx *sql.Tx, query string, values, out []any) error { } /* -QueryRowTxSQL gets a row from the db with the values in values[] and fills the respective pointers in out[] -Automatically escapes the given values and caches the query +QueryRowTxSQL gets a row from the db with the values in values[] and fills the respective pointers in out[]. +Deprecated: Use QueryRow instead with a transaction in the RequestOptions Example: @@ -261,8 +299,8 @@ func QueryRowTxSQL(tx *sql.Tx, query string, values, out []any) error { } /* -QuerySQL gets all rows from the db with the values in values[] and fills the respective pointers in out[] -Automatically escapes the given values and caches the query +QuerySQL gets all rows from the db with the values in values[] and fills the respective pointers in out[]. +Deprecated: Use Query instead Example: @@ -285,7 +323,8 @@ func QuerySQL(query string, a ...any) (*sql.Rows, error) { /* QueryContextSQL queries the database with a prepared statement and the given parameters, using the given context -for a deadline +for a deadline. +Deprecated: Use Query instead with an optional context/transaction in the RequestOptions Example: @@ -317,7 +356,9 @@ func QueryTimeoutSQL(tx *sql.Tx, query string, a ...any) (*sql.Rows, context.Can /* QueryTxSQL gets all rows from the db using the transaction tx with the values in values[] and fills the -respective pointers in out[]. Automatically escapes the given values and caches the query +respective pointers in out[]. +Deprecated: Use Query instead with a transaction in the RequestOptions + Example: tx, err := BeginTx() @@ -347,6 +388,7 @@ func QueryTxSQL(tx *sql.Tx, query string, a ...any) (*sql.Rows, error) { return rows, stmt.Close() } +// ParseSQLTimeString attempts to parse a string into a time.Time object using the known SQL date/time formats func ParseSQLTimeString(str string) (time.Time, error) { var t time.Time var err error @@ -359,22 +401,10 @@ func ParseSQLTimeString(str string) (time.Time, error) { } // getLatestID returns the latest inserted id column value from the given table -func getLatestID(tableName string, tx *sql.Tx) (id int, err error) { +func getLatestID(opts *RequestOptions, tableName string) (id int, err error) { + opts = setupOptions(opts) query := `SELECT MAX(id) FROM ` + tableName - if tx != nil { - var stmt *sql.Stmt - stmt, err = PrepareSQL(query, tx) - if err != nil { - return 0, err - } - defer stmt.Close() - if err = stmt.QueryRow().Scan(&id); err != nil { - return - } - err = stmt.Close() - } else { - err = QueryRowSQL(query, nil, []any{&id}) - } + QueryRow(opts, query, nil, []any{&id}) return } @@ -393,7 +423,7 @@ func doesTableExist(tableName string) (bool, error) { } var count int - err := QueryRowSQL(existQuery, []any{config.GetSystemCriticalConfig().DBprefix + tableName}, []any{&count}) + err := QueryRow(nil, existQuery, []any{config.GetSystemCriticalConfig().DBprefix + tableName}, []any{&count}) if err != nil { return false, err } @@ -404,7 +434,7 @@ func doesTableExist(tableName string) (bool, error) { func GetComponentVersion(componentKey string) (int, error) { const sql = `SELECT version FROM DBPREFIXdatabase_version WHERE component = ?` var version int - err := QueryRowSQL(sql, []any{componentKey}, []any{&version}) + err := QueryRow(nil, sql, []any{componentKey}, []any{&version}) return version, err } @@ -434,7 +464,7 @@ func doesGochanPrefixTableExist() (bool, error) { } var count int - err := QueryRowSQL(prefixTableExist, []any{}, []any{&count}) + err := QueryRow(nil, prefixTableExist, []any{}, []any{&count}) if err != nil && err != sql.ErrNoRows { return false, err } From 7ceda2b218ab158312f2e74103398792e369a6ad Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 5 Feb 2025 23:32:12 -0800 Subject: [PATCH 094/122] Fully refactor all (or at least most) uses of ExecSQL, QuerySQL, and QueryRowSQL, QueryContextSQL, etc with their replacement functions --- .../internal/common/sqlutil.go | 2 +- .../internal/gcupdate/gcupdate.go | 2 +- .../internal/pre2021/announcements.go | 4 +-- .../internal/pre2021/announcements_test.go | 2 +- cmd/gochan-migration/internal/pre2021/bans.go | 4 +-- .../internal/pre2021/bans_test.go | 2 +- .../internal/pre2021/posts_test.go | 12 ++++----- .../internal/pre2021/pre2021.go | 4 +-- .../internal/pre2021/staff.go | 10 +++---- cmd/gochan/deleteposts.go | 22 ++++++++++------ pkg/gcsql/boards.go | 26 ++++++++++--------- pkg/gcsql/initsql/init.go | 2 +- pkg/gcsql/threads.go | 5 +++- pkg/gcsql/util.go | 17 +++++++++--- pkg/manage/actionsAdminPerm.go | 2 +- pkg/manage/actionsModPerm.go | 3 +-- pkg/manage/util.go | 12 +++++++-- 17 files changed, 79 insertions(+), 52 deletions(-) diff --git a/cmd/gochan-migration/internal/common/sqlutil.go b/cmd/gochan-migration/internal/common/sqlutil.go index bfacccce..255a42fd 100644 --- a/cmd/gochan-migration/internal/common/sqlutil.go +++ b/cmd/gochan-migration/internal/common/sqlutil.go @@ -88,7 +88,7 @@ func RunSQLFile(path string, db *gcsql.GCDB) error { for _, statement := range sqlArr { statement = strings.TrimSpace(statement) if len(statement) > 0 { - if _, err = db.ExecSQL(statement); err != nil { + if _, err = db.Exec(nil, statement); err != nil { return err } } diff --git a/cmd/gochan-migration/internal/gcupdate/gcupdate.go b/cmd/gochan-migration/internal/gcupdate/gcupdate.go index 1059cc40..faaf69b8 100644 --- a/cmd/gochan-migration/internal/gcupdate/gcupdate.go +++ b/cmd/gochan-migration/internal/gcupdate/gcupdate.go @@ -36,7 +36,7 @@ func (dbu *GCDatabaseUpdater) Init(options *common.MigrationOptions) error { func (dbu *GCDatabaseUpdater) IsMigrated() (bool, error) { var currentDatabaseVersion int - err := dbu.db.QueryRowSQL(`SELECT version FROM DBPREFIXdatabase_version WHERE component = 'gochan'`, nil, + err := dbu.db.QueryRow(nil, "SELECT version FROM DBPREFIXdatabase_version WHERE component = 'gochan'", nil, []any{¤tDatabaseVersion}) if err != nil { return false, err diff --git a/cmd/gochan-migration/internal/pre2021/announcements.go b/cmd/gochan-migration/internal/pre2021/announcements.go index aecbf909..705d7595 100644 --- a/cmd/gochan-migration/internal/pre2021/announcements.go +++ b/cmd/gochan-migration/internal/pre2021/announcements.go @@ -16,7 +16,7 @@ func (m *Pre2021Migrator) MigrateAnnouncements() error { errEv := common.LogError() defer errEv.Discard() - rows, err := m.db.QuerySQL(announcementsQuery) + rows, err := m.db.Query(nil, announcementsQuery) if err != nil { errEv.Err(err).Caller().Msg("Failed to get announcements") return err @@ -49,7 +49,7 @@ func (m *Pre2021Migrator) MigrateAnnouncements() error { errEv.Err(err).Caller().Str("staff", announcement.oldPoster).Msg("Failed to get staff ID") return err } - if _, err = gcsql.ExecSQL( + if _, err = gcsql.Exec(nil, "INSERT INTO DBPREFIXannouncements(staff_id,subject,message,timestamp) values(?,?,?,?)", announcement.StaffID, announcement.Subject, announcement.Message, announcement.Timestamp, ); err != nil { diff --git a/cmd/gochan-migration/internal/pre2021/announcements_test.go b/cmd/gochan-migration/internal/pre2021/announcements_test.go index 99684351..f91c2393 100644 --- a/cmd/gochan-migration/internal/pre2021/announcements_test.go +++ b/cmd/gochan-migration/internal/pre2021/announcements_test.go @@ -27,6 +27,6 @@ func TestMigrateAnnouncements(t *testing.T) { } var numAnnouncements int - assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXannouncements WHERE staff_id > 0", nil, []any{&numAnnouncements})) + assert.NoError(t, gcsql.QueryRow(nil, "SELECT COUNT(*) FROM DBPREFIXannouncements WHERE staff_id > 0", nil, []any{&numAnnouncements})) assert.Equal(t, 2, numAnnouncements, "Expected to have two announcement") } diff --git a/cmd/gochan-migration/internal/pre2021/bans.go b/cmd/gochan-migration/internal/pre2021/bans.go index 40d8bd5c..5a0bfd10 100644 --- a/cmd/gochan-migration/internal/pre2021/bans.go +++ b/cmd/gochan-migration/internal/pre2021/bans.go @@ -53,7 +53,7 @@ func (m *Pre2021Migrator) migrateBansInPlace() error { for _, stmt := range statements { stmt = strings.TrimSpace(stmt) if strings.HasPrefix(stmt, "CREATE TABLE DBPREFIXip_ban") || strings.HasPrefix(stmt, "CREATE TABLE DBPREFIXfilter") { - _, err = gcsql.ExecSQL(stmt) + _, err = gcsql.Exec(nil, stmt) if err != nil { errEv.Err(err).Caller(). Str("statement", stmt). @@ -99,7 +99,7 @@ func (m *Pre2021Migrator) migrateBansToNewDB() error { } defer tx.Rollback() - rows, err := m.db.QuerySQL(bansQuery) + rows, err := m.db.Query(nil, bansQuery) if err != nil { errEv.Err(err).Caller().Msg("Failed to get bans") return err diff --git a/cmd/gochan-migration/internal/pre2021/bans_test.go b/cmd/gochan-migration/internal/pre2021/bans_test.go index 43f0a737..dceeeeb9 100644 --- a/cmd/gochan-migration/internal/pre2021/bans_test.go +++ b/cmd/gochan-migration/internal/pre2021/bans_test.go @@ -42,7 +42,7 @@ func validateBanMigration(t *testing.T) { assert.NotZero(t, bans[0].StaffID, "Expected ban staff ID field to be set") var numInvalidBans int - assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXip_ban WHERE message = ?", []any{"Full ban on 8.8.0.0/16"}, []any{&numInvalidBans})) + assert.NoError(t, gcsql.QueryRow(nil, "SELECT COUNT(*) FROM DBPREFIXip_ban WHERE message = ?", []any{"Full ban on 8.8.0.0/16"}, []any{&numInvalidBans})) assert.Equal(t, 0, numInvalidBans, "Expected the invalid test to not be migrated") filters, err := gcsql.GetAllFilters(gcsql.TrueOrFalse) diff --git a/cmd/gochan-migration/internal/pre2021/posts_test.go b/cmd/gochan-migration/internal/pre2021/posts_test.go index f4b605b2..af08de97 100644 --- a/cmd/gochan-migration/internal/pre2021/posts_test.go +++ b/cmd/gochan-migration/internal/pre2021/posts_test.go @@ -19,7 +19,7 @@ func TestMigratePosts(t *testing.T) { } var numThreads int - if !assert.NoError(t, migrator.db.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXposts WHERE parentid = 0 AND deleted_timestamp IS NULL", nil, []any{&numThreads}), "Failed to get number of threads") { + if !assert.NoError(t, migrator.db.QueryRow(nil, "SELECT COUNT(*) FROM DBPREFIXposts WHERE parentid = 0 AND deleted_timestamp IS NULL", nil, []any{&numThreads}), "Failed to get number of threads") { t.FailNow() } assert.Equal(t, 2, numThreads, "Expected to have two threads pre-migration") @@ -32,27 +32,27 @@ func TestMigratePosts(t *testing.T) { func validatePostMigration(t *testing.T) { var numThreads int - if !assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numThreads}), "Failed to get number of threads") { + if !assert.NoError(t, gcsql.QueryRow(nil, "SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numThreads}), "Failed to get number of threads") { t.FailNow() } assert.Equal(t, 2, numThreads, "Expected to have two threads pre-migration") var numUploadPosts int - assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXfiles", nil, []any{&numUploadPosts})) + assert.NoError(t, gcsql.QueryRow(nil, "SELECT COUNT(*) FROM DBPREFIXfiles", nil, []any{&numUploadPosts})) assert.Equal(t, 1, numUploadPosts, "Expected to have 1 upload post") var ip string - assert.NoError(t, gcsql.QueryRowSQL("SELECT IP_NTOA FROM DBPREFIXposts WHERE id = 1", nil, []any{&ip})) + assert.NoError(t, gcsql.QueryRow(nil, "SELECT IP_NTOA FROM DBPREFIXposts WHERE id = 1", nil, []any{&ip})) assert.Equal(t, "192.168.56.1", ip, "Expected to have the correct IP address") var numMigratedThreads int - if !assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numMigratedThreads}), "Failed to get number of migrated threads") { + if !assert.NoError(t, gcsql.QueryRow(nil, "SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numMigratedThreads}), "Failed to get number of migrated threads") { t.FailNow() } assert.Equal(t, 2, numMigratedThreads, "Expected to have three migrated threads") var locked bool - if !assert.NoError(t, gcsql.QueryRowSQL("SELECT locked FROM DBPREFIXthreads WHERE id = 1", nil, []any{&locked})) { + if !assert.NoError(t, gcsql.QueryRow(nil, "SELECT locked FROM DBPREFIXthreads WHERE id = 1", nil, []any{&locked})) { t.FailNow() } assert.True(t, locked, "Expected thread ID 1 to be locked") diff --git a/cmd/gochan-migration/internal/pre2021/pre2021.go b/cmd/gochan-migration/internal/pre2021/pre2021.go index 8562569e..296f5a1a 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021.go @@ -72,12 +72,12 @@ func (m *Pre2021Migrator) renameTablesForInPlace() error { var err error errEv := common.LogError() defer errEv.Discard() - if _, err = m.db.ExecSQL("DROP TABLE DBPREFIXinfo"); err != nil { + if _, err = m.db.Exec(nil, "DROP TABLE DBPREFIXinfo"); err != nil { errEv.Err(err).Caller().Msg("Error dropping info table") return err } for _, table := range renameTables { - if _, err = m.db.ExecSQL(fmt.Sprintf(renameTableStatementTemplate, table, table)); err != nil { + if _, err = m.db.Exec(nil, fmt.Sprintf(renameTableStatementTemplate, table, table)); err != nil { errEv.Caller().Err(err). Str("table", table). Msg("Error renaming table") diff --git a/cmd/gochan-migration/internal/pre2021/staff.go b/cmd/gochan-migration/internal/pre2021/staff.go index f1333743..65379936 100644 --- a/cmd/gochan-migration/internal/pre2021/staff.go +++ b/cmd/gochan-migration/internal/pre2021/staff.go @@ -26,13 +26,13 @@ func (m *Pre2021Migrator) getMigrationUser(errEv *zerolog.Event) (*gcsql.Staff, Username: "pre2021-migration" + gcutil.RandomString(8), AddedOn: time.Now(), } - _, err := gcsql.ExecSQL("INSERT INTO DBPREFIXstaff(username,password_checksum,global_rank,is_active) values(?,'',0,0)", user.Username) + _, err := gcsql.Exec(nil, "INSERT INTO DBPREFIXstaff(username,password_checksum,global_rank,is_active) values(?,'',0,0)", user.Username) if err != nil { errEv.Err(err).Caller().Str("username", user.Username).Msg("Failed to create migration user") return nil, err } - if err = gcsql.QueryRowSQL("SELECT id FROM DBPREFIXstaff WHERE username = ?", []any{user.Username}, []any{&user.ID}); err != nil { + if err = gcsql.QueryRow(nil, "SELECT id FROM DBPREFIXstaff WHERE username = ?", []any{user.Username}, []any{&user.ID}); err != nil { errEv.Err(err).Caller().Str("username", user.Username).Msg("Failed to get migration user ID") return nil, err } @@ -50,7 +50,7 @@ func (m *Pre2021Migrator) MigrateStaff() error { return err } - rows, err := m.db.QuerySQL(staffQuery) + rows, err := m.db.Query(nil, staffQuery) if err != nil { errEv.Err(err).Caller().Msg("Failed to get ban rows") return err @@ -73,7 +73,7 @@ func (m *Pre2021Migrator) MigrateStaff() error { staff.ID = newStaff.ID } else if errors.Is(err, gcsql.ErrUnrecognizedUsername) { // staff doesn't exist, create it (with invalid checksum to be updated by the admin) - if _, err := gcsql.ExecSQL( + if _, err := gcsql.Exec(nil, "INSERT INTO DBPREFIXstaff(username,password_checksum,global_rank,added_on,last_login,is_active) values(?,'',?,?,?,1)", staff.Username, staff.Rank, staff.AddedOn, staff.LastLogin, ); err != nil { @@ -102,7 +102,7 @@ func (m *Pre2021Migrator) MigrateStaff() error { Msg("Failed to get board ID") return err } - if _, err = gcsql.ExecSQL("INSERT INTO DBPREFIXboard_staff(board_id,staff_id) VALUES(?,?)", boardID, staff.ID); err != nil { + if _, err = gcsql.Exec(nil, "INSERT INTO DBPREFIXboard_staff(board_id,staff_id) VALUES(?,?)", boardID, staff.ID); err != nil { errEv.Err(err).Caller(). Str("username", staff.Username). Str("board", board). diff --git a/cmd/gochan/deleteposts.go b/cmd/gochan/deleteposts.go index 5aad2733..01506cef 100644 --- a/cmd/gochan/deleteposts.go +++ b/cmd/gochan/deleteposts.go @@ -125,16 +125,19 @@ func getAllPostsToDelete(postIDs []any, fileOnly bool) ([]delPost, []any, error) query = "SELECT post_id, thread_id, op_id, is_top_post, filename, dir FROM DBPREFIXv_posts_to_delete WHERE post_id IN " + setPart + " OR thread_id IN (SELECT thread_id from DBPREFIXposts op WHERE op_id IN " + setPart + " AND is_top_post)" } - rows, err := gcsql.QuerySQL(query, params...) + rows, cancel, err := gcsql.QueryTimeoutSQL(nil, query, params...) if err != nil { return nil, nil, err } + defer func() { + rows.Close() + cancel() + }() var posts []delPost var postIDsAny []any for rows.Next() { var post delPost if err = rows.Scan(&post.postID, &post.threadID, &post.opID, &post.isOP, &post.filename, &post.boardDir); err != nil { - rows.Close() return nil, nil, err } posts = append(posts, post) @@ -258,7 +261,7 @@ func validatePostPasswords(posts []any, passwordMD5 string) (bool, error) { params = append(params, posts[p]) } - err := gcsql.QueryRowSQL(queryPosts, params, []any{&count}) + err := gcsql.QueryRow(nil, queryPosts, params, []any{&count}) return count == len(posts), err } @@ -281,21 +284,24 @@ func markPostsAsDeleted(posts []any, request *http.Request, writer http.Response defer cancel() tx, err := gcsql.BeginContextTx(ctx) + opts := &gcsql.RequestOptions{Context: ctx, Tx: tx, Cancel: cancel} wantsJSON := serverutil.IsRequestingJSON(request) if err != nil { + errEv.Err(err).Caller().Msg("Unable to start deletion transaction") serveError(writer, "Unable to delete posts", http.StatusInternalServerError, wantsJSON, errEv.Err(err).Caller()) return false } defer tx.Rollback() - const postsError = "Unable to mark post(s) as deleted" - const threadsError = "Unable to mark thread(s) as deleted" - if _, err = gcsql.ExecTxSQL(tx, deletePostsSQL, posts...); err != nil { + const postsError = "Unable to delete post(s)" + const threadsError = "Unable to delete thread(s)" + if _, err = gcsql.Exec(opts, deletePostsSQL, posts...); err != nil { serveError(writer, postsError, http.StatusInternalServerError, wantsJSON, errEv.Err(err).Caller()) return false } - if _, err = gcsql.ExecTxSQL(tx, deleteThreadSQL, posts...); err != nil { + if _, err = gcsql.Exec(opts, deleteThreadSQL, posts...); err != nil { + errEv.Err(err).Caller().Msg("Unable to mark thread(s) as deleted") serveError(writer, threadsError, http.StatusInternalServerError, wantsJSON, errEv.Err(err).Caller()) return false } @@ -351,7 +357,7 @@ func deletePostFiles(posts []delPost, deleteIDs []any, permDelete bool, request http.StatusInternalServerError, wantsJSON, errEv.Array("errors", errArr)) return false } - _, err = gcsql.ExecSQL(deleteFilesSQL, deleteIDs...) + _, err = gcsql.ExecTimeoutSQL(nil, deleteFilesSQL, deleteIDs...) if err != nil { serveError(writer, "Unable to delete file entries from database", http.StatusInternalServerError, wantsJSON, errEv.Err(err).Caller()) diff --git a/pkg/gcsql/boards.go b/pkg/gcsql/boards.go index 87cbb281..3f325b6d 100644 --- a/pkg/gcsql/boards.go +++ b/pkg/gcsql/boards.go @@ -36,18 +36,18 @@ var ( ) // DoesBoardExistByID returns a bool indicating whether a board with a given id exists -func DoesBoardExistByID(ID int) bool { - const query = `SELECT COUNT(id) FROM DBPREFIXboards WHERE id = ?` +func DoesBoardExistByID(ID int, requestOptions ...*RequestOptions) bool { + opts := setupOptionsWithTimeout(requestOptions...) var count int - QueryRowSQL(query, []any{ID}, []any{&count}) + QueryRow(opts, "SELECT COUNT(id) FROM DBPREFIXboards WHERE id = ?", []any{ID}, []any{&count}) return count > 0 } // DoesBoardExistByDir returns a bool indicating whether a board with a given directory exists -func DoesBoardExistByDir(dir string) bool { - const query = `SELECT COUNT(dir) FROM DBPREFIXboards WHERE dir = ?` +func DoesBoardExistByDir(dir string, requestOpts ...*RequestOptions) bool { + opts := setupOptionsWithTimeout(requestOpts...) var count int - QueryRowSQL(query, []any{dir}, []any{&count}) + QueryRow(opts, "SELECT COUNT(dir) FROM DBPREFIXboards WHERE dir = ?", []any{dir}, []any{&count}) return count > 0 } @@ -108,10 +108,10 @@ func GetBoardDirFromPostID(postID int) (string, error) { return boardURI, err } -func getBoardBase(where string, whereParameters []interface{}) (*Board, error) { +func getBoardBase(requestOptions *RequestOptions, where string, whereParameters ...any) (*Board, error) { query := selectBoardsBaseSQL + where board := new(Board) - err := QueryRowTimeoutSQL(nil, query, whereParameters, []any{ + err := QueryRow(requestOptions, query, whereParameters, []any{ &board.ID, &board.SectionID, &board.URI, &board.Dir, &board.NavbarPosition, &board.Title, &board.Subtitle, &board.Description, &board.MaxFilesize, &board.MaxThreads, &board.DefaultStyle, &board.Locked, &board.CreatedAt, &board.AnonymousName, &board.ForceAnonymous, &board.AutosageAfter, &board.NoImagesAfter, @@ -124,13 +124,15 @@ func getBoardBase(where string, whereParameters []interface{}) (*Board, error) { } // GetBoardFromID returns the board corresponding to a given id -func GetBoardFromID(id int) (*Board, error) { - return getBoardBase("WHERE DBPREFIXboards.id = ?", []any{id}) +func GetBoardFromID(id int, requestOptions ...*RequestOptions) (*Board, error) { + opts := setupOptionsWithTimeout(requestOptions...) + return getBoardBase(opts, "WHERE DBPREFIXboards.id = ?", id) } // GetBoardFromDir returns the board corresponding to a given dir -func GetBoardFromDir(dir string) (*Board, error) { - return getBoardBase("WHERE DBPREFIXboards.dir = ?", []any{dir}) +func GetBoardFromDir(dir string, requestOptions ...*RequestOptions) (*Board, error) { + opts := setupOptionsWithTimeout(requestOptions...) + return getBoardBase(opts, "WHERE DBPREFIXboards.dir = ?", dir) } // GetIDFromDir returns the id of the board with the given dir value diff --git a/pkg/gcsql/initsql/init.go b/pkg/gcsql/initsql/init.go index ccbbbef6..a1434cb4 100644 --- a/pkg/gcsql/initsql/init.go +++ b/pkg/gcsql/initsql/init.go @@ -97,7 +97,7 @@ func getBoardDefaultStyleTmplFunc(dir string) string { return boardCfg.DefaultStyle } var defaultStyle string - err := gcsql.QueryRowSQL(`SELECT default_style FROM DBPREFIXboards WHERE dir = ?`, + err := gcsql.QueryRowTimeoutSQL(nil, "SELECT default_style FROM DBPREFIXboards WHERE dir = ?", []any{dir}, []any{&defaultStyle}) if err != nil || defaultStyle == "" { gcutil.LogError(err).Caller(). diff --git a/pkg/gcsql/threads.go b/pkg/gcsql/threads.go index 99f65446..6e66e6dc 100644 --- a/pkg/gcsql/threads.go +++ b/pkg/gcsql/threads.go @@ -33,7 +33,10 @@ func CreateThread(requestOptions *RequestOptions, boardID int, locked bool, stic if _, err = Exec(requestOptions, insertQuery, boardID, locked, stickied, anchored, cyclic); err != nil { return 0, err } - return threadID, QueryRow(requestOptions, "SELECT MAX(id) FROM DBPREFIXthreads", nil, []any{&threadID}) + if err = QueryRow(requestOptions, "SELECT MAX(id) FROM DBPREFIXthreads", nil, []any{&threadID}); err != nil { + return 0, err + } + return threadID, nil } // GetThread returns a a thread object from the database, given its ID diff --git a/pkg/gcsql/util.go b/pkg/gcsql/util.go index a82ad968..871008dd 100644 --- a/pkg/gcsql/util.go +++ b/pkg/gcsql/util.go @@ -69,17 +69,26 @@ func setupOptions(opts ...*RequestOptions) *RequestOptions { return opts[0] } -// Query is a wrapper for QueryContextSQL that uses the given options, or defaults to a background context if nil +func setupOptionsWithTimeout(opts ...*RequestOptions) *RequestOptions { + withoutContext := len(opts) == 0 || opts[0] == nil || opts[0].Context == nil + requestOptions := setupOptions(opts...) + if withoutContext { + requestOptions.Context, requestOptions.Cancel = context.WithTimeout(context.Background(), gcdb.defaultTimeout) + } + return requestOptions +} + +// Query is a function for querying rows from the configured database, using the given options, or defaults to a background context if nil func Query(opts *RequestOptions, query string, a ...any) (*sql.Rows, error) { return gcdb.Query(opts, query, a...) } -// QueryRow is a wrapper for QueryRowContextSQL that uses the given options, or defaults to a background context if nil +// QueryRow is a function for querying a single row from the configured database, using the given options, or defaults to a background context if nil func QueryRow(opts *RequestOptions, query string, values, out []any) error { return gcdb.QueryRow(opts, query, values, out) } -// Exec is a wrapper for ExecContextSQL that uses the given options, or defaults to a background context if nil +// Exec is a function for executing a statement with the configured database, using the given options, or defaults to a background context if nil func Exec(opts *RequestOptions, query string, values ...any) (sql.Result, error) { return gcdb.Exec(opts, query, values...) } @@ -269,7 +278,7 @@ func QueryRowContextSQL(ctx context.Context, tx *sql.Tx, query string, values, o // QueryRowTimeoutSQL is a helper function for querying a single row with the configured default timeout. // It creates a context with the default timeout to only be used for this query and then disposed. -// It should only be used by a function that does a single SQL query, otherwise use QueryRowContextSQL +// It should only be used by a function that does a single SQL query, otherwise use QueryRow with a context. func QueryRowTimeoutSQL(tx *sql.Tx, query string, values, out []any) error { ctx, cancel := context.WithTimeout(context.Background(), gcdb.defaultTimeout) defer cancel() diff --git a/pkg/manage/actionsAdminPerm.go b/pkg/manage/actionsAdminPerm.go index d2d0672c..d8d51260 100644 --- a/pkg/manage/actionsAdminPerm.go +++ b/pkg/manage/actionsAdminPerm.go @@ -358,7 +358,7 @@ func fixThumbnailsCallback(_ http.ResponseWriter, request *http.Request, _ *gcsq if board != "" { const query = `SELECT id, op, filename, is_spoilered, width, height, thumbnail_width, thumbnail_height FROM DBPREFIXv_upload_info WHERE dir = ? ORDER BY created_on DESC` - rows, err := gcsql.QuerySQL(query, board) + rows, err := gcsql.Query(nil, query, board) if err != nil { return "", err } diff --git a/pkg/manage/actionsModPerm.go b/pkg/manage/actionsModPerm.go index 75331ca9..8a096770 100644 --- a/pkg/manage/actionsModPerm.go +++ b/pkg/manage/actionsModPerm.go @@ -415,8 +415,7 @@ func reportsCallback(_ http.ResponseWriter, request *http.Request, staff *gcsql. Bool("blocked", block != ""). Msg("Report cleared") } - rows, err := gcsql.QuerySQL(`SELECT id, - handled_by_staff_id as staff_id, + rows, err := gcsql.Query(nil, `SELECT id, handled_by_staff_id as staff_id, (SELECT username FROM DBPREFIXstaff WHERE id = DBPREFIXreports.handled_by_staff_id) as staff_user, post_id, IP_NTOA, reason, is_cleared from DBPREFIXreports WHERE is_cleared = FALSE`) if err != nil { diff --git a/pkg/manage/util.go b/pkg/manage/util.go index 47d1f70e..63135579 100644 --- a/pkg/manage/util.go +++ b/pkg/manage/util.go @@ -2,6 +2,7 @@ package manage import ( "bytes" + "context" "database/sql" "errors" "net/http" @@ -191,11 +192,15 @@ func getAllStaffNopass(activeOnly bool) ([]gcsql.Staff, error) { if activeOnly { query += " WHERE is_active" } - rows, err := gcsql.QuerySQL(query) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(config.GetSQLConfig().DBTimeoutSeconds)*time.Second) + rows, err := gcsql.Query(&gcsql.RequestOptions{Context: ctx, Cancel: cancel}, query) if err != nil { return nil, err } - defer rows.Close() + defer func() { + rows.Close() + cancel() + }() var staff []gcsql.Staff for rows.Next() { var s gcsql.Staff @@ -205,6 +210,9 @@ func getAllStaffNopass(activeOnly bool) ([]gcsql.Staff, error) { } staff = append(staff, s) } + if err = rows.Close(); err != nil { + return nil, err + } return staff, nil } From a31a00176401ec17236b65f2742df04640bfa16a Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Fri, 7 Feb 2025 16:14:04 -0800 Subject: [PATCH 095/122] Add spoiler bbcode tag --- pkg/posting/formatting.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/posting/formatting.go b/pkg/posting/formatting.go index 52c66eab..90701646 100644 --- a/pkg/posting/formatting.go +++ b/pkg/posting/formatting.go @@ -14,8 +14,9 @@ import ( ) var ( - msgfmtr *MessageFormatter - urlRE = regexp.MustCompile(`https?://(\S+)`) + msgfmtr *MessageFormatter + urlRE = regexp.MustCompile(`https?://(\S+)`) + unsetBBcodeTags = []string{"center", "color", "img", "quote", "size"} ) // InitPosting prepares the formatter and the temp post pruner @@ -35,12 +36,12 @@ type MessageFormatter struct { func (mf *MessageFormatter) Init() { mf.bbCompiler = bbcode.NewCompiler(true, true) - mf.bbCompiler.SetTag("center", nil) - // mf.bbCompiler.SetTag("code", nil) - mf.bbCompiler.SetTag("color", nil) - mf.bbCompiler.SetTag("img", nil) - mf.bbCompiler.SetTag("quote", nil) - mf.bbCompiler.SetTag("size", nil) + for _, tag := range unsetBBcodeTags { + mf.bbCompiler.SetTag(tag, nil) + } + mf.bbCompiler.SetTag("?", func(bn *bbcode.BBCodeNode) (*bbcode.HTMLTag, bool) { + return &bbcode.HTMLTag{Name: "span", Attrs: map[string]string{"class": "spoiler"}}, true + }) mf.linkFixer = strings.NewReplacer( "[url=[url]", "[url=", "[/url][/url]", "[/url]", From 5d98639daadbb986de68d0918d79356943aa40aa Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Fri, 7 Feb 2025 16:26:35 -0800 Subject: [PATCH 096/122] Add hide block bbcode --- frontend/sass/global/_img.scss | 7 +++++++ html/css/global.css | 8 ++++++++ pkg/posting/formatting.go | 3 +++ 3 files changed, 18 insertions(+) diff --git a/frontend/sass/global/_img.scss b/frontend/sass/global/_img.scss index 7cb30546..c5175db0 100755 --- a/frontend/sass/global/_img.scss +++ b/frontend/sass/global/_img.scss @@ -218,4 +218,11 @@ div.inlinepostprev { overflow: hidden; margin:8px; padding: 4px 8px; +} + +.hideblock { + border: 2px solid black; +} +.hideblock.hidden { + display: none; } \ No newline at end of file diff --git a/html/css/global.css b/html/css/global.css index e2da2500..2e46cd01 100644 --- a/html/css/global.css +++ b/html/css/global.css @@ -220,6 +220,14 @@ div.inlinepostprev { padding: 4px 8px; } +.hideblock { + border: 2px solid black; +} + +.hideblock.hidden { + display: none; +} + div.section-block { margin-bottom: 8px; } diff --git a/pkg/posting/formatting.go b/pkg/posting/formatting.go index 90701646..cdc0e725 100644 --- a/pkg/posting/formatting.go +++ b/pkg/posting/formatting.go @@ -42,6 +42,9 @@ func (mf *MessageFormatter) Init() { mf.bbCompiler.SetTag("?", func(bn *bbcode.BBCodeNode) (*bbcode.HTMLTag, bool) { return &bbcode.HTMLTag{Name: "span", Attrs: map[string]string{"class": "spoiler"}}, true }) + mf.bbCompiler.SetTag("hide", func(bn *bbcode.BBCodeNode) (*bbcode.HTMLTag, bool) { + return &bbcode.HTMLTag{Name: "div", Attrs: map[string]string{"class": "hideblock hidden"}}, true + }) mf.linkFixer = strings.NewReplacer( "[url=[url]", "[url=", "[/url][/url]", "[/url]", From 21a05e5d2518fdfb4b8a7abb0b2b1bee6bbac5e8 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Fri, 7 Feb 2025 16:35:12 -0800 Subject: [PATCH 097/122] Add hidden block toggling button --- frontend/sass/global/_img.scss | 3 +++ frontend/ts/gochan.ts | 2 ++ frontend/ts/postutil.ts | 13 +++++++++++++ html/css/global.css | 4 ++++ 4 files changed, 22 insertions(+) diff --git a/frontend/sass/global/_img.scss b/frontend/sass/global/_img.scss index c5175db0..bd30be6d 100755 --- a/frontend/sass/global/_img.scss +++ b/frontend/sass/global/_img.scss @@ -225,4 +225,7 @@ div.inlinepostprev { } .hideblock.hidden { display: none; +} +.hideblock-button { + display: block; } \ No newline at end of file diff --git a/frontend/ts/gochan.ts b/frontend/ts/gochan.ts index 88508774..5d49040d 100755 --- a/frontend/ts/gochan.ts +++ b/frontend/ts/gochan.ts @@ -14,6 +14,7 @@ import { initFlags } from "./dom/flags"; import { initQR } from "./dom/qr"; import { getBooleanStorageVal } from "./storage"; import { updateBrowseButton } from "./dom/uploaddata"; +import { prepareHideBlocks } from "./postutil"; import "./management/filters"; export function toTop() { @@ -43,6 +44,7 @@ $(() => { const passwordText = $("input#postpassword").val(); $("input#delete-password").val(passwordText); + prepareHideBlocks(); setPageBanner(); if(pageThread.board !== "") { diff --git a/frontend/ts/postutil.ts b/frontend/ts/postutil.ts index d089bdd3..0afa32a8 100755 --- a/frontend/ts/postutil.ts +++ b/frontend/ts/postutil.ts @@ -227,6 +227,19 @@ function selectedText() { return window.getSelection().toString(); } +export function prepareHideBlocks() { + $("div.hideblock").each((_i,el) => { + const $el = $(el); + const $button = $("
- +
diff --git a/pkg/gctemplates/templatetests/templatecases_test.go b/pkg/gctemplates/templatetests/templatecases_test.go index 3206fcbe..d63a1a64 100644 --- a/pkg/gctemplates/templatetests/templatecases_test.go +++ b/pkg/gctemplates/templatetests/templatecases_test.go @@ -36,7 +36,7 @@ const ( `` + `
` - footer = `` + footer = `` ) var ( @@ -165,7 +165,7 @@ var ( }, }, expectedOutput: normalBanHeader + - `You are banned from posting onall boardsfor the following reason:

ban message goes here

Your ban was placed on and will expire on .
Your IP address is192.168.56.1.

You may appeal this ban:

`, + `You are banned from posting onall boardsfor the following reason:

ban message goes here

Your ban was placed on and will expire on .
Your IP address is192.168.56.1.

You may appeal this ban:

`, }, { desc: "unappealable temporary ban", diff --git a/templates/captcha.html b/templates/captcha.html index d89802e7..2d1c5a97 100644 --- a/templates/captcha.html +++ b/templates/captcha.html @@ -24,9 +24,9 @@ - + diff --git a/templates/page_footer.html b/templates/page_footer.html index 1558dd32..11e12290 100644 --- a/templates/page_footer.html +++ b/templates/page_footer.html @@ -1,6 +1,6 @@ - + From 5ae229d61e6b29b40e7fdfc595f8dc17841b27d0 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Tue, 18 Feb 2025 13:16:07 -0800 Subject: [PATCH 114/122] Add LoadPlugins test --- pkg/gcplugin/gcplugin.go | 21 +-------------------- pkg/gcplugin/lua_test.go | 14 +++++++++++++- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/pkg/gcplugin/gcplugin.go b/pkg/gcplugin/gcplugin.go index 89438578..b1e8d44c 100644 --- a/pkg/gcplugin/gcplugin.go +++ b/pkg/gcplugin/gcplugin.go @@ -45,6 +45,7 @@ func initLua() { func ClosePlugins() { if lState != nil { lState.Close() + lState = nil } } @@ -112,17 +113,6 @@ func preloadLua() { lState.SetGlobal("_GOCHAN_VERSION", lua.LString(config.GetVersion().String())) } -func registerEventFunction(name string, fn *lua.LFunction) { - switch name { - case "onStartup": - fallthrough - case "onPost": - fallthrough - case "onDelete": - eventPlugins[name] = append(eventPlugins[name], fn) - } -} - func LoadPlugins(paths []string) error { var err error initLua() @@ -134,15 +124,6 @@ func LoadPlugins(paths []string) error { if err = lState.DoFile(pluginPath); err != nil { return err } - pluginTable := lState.NewTable() - pluginTable.ForEach(func(key, val lua.LValue) { - keyStr := key.String() - fn, ok := val.(*lua.LFunction) - if !ok { - return - } - registerEventFunction(keyStr, fn) - }) case ".so": nativePlugin, err := plugin.Open(pluginPath) if err != nil { diff --git a/pkg/gcplugin/lua_test.go b/pkg/gcplugin/lua_test.go index df9906f5..3738b258 100644 --- a/pkg/gcplugin/lua_test.go +++ b/pkg/gcplugin/lua_test.go @@ -6,6 +6,7 @@ import ( "github.com/gochan-org/gochan/pkg/config" "github.com/gochan-org/gochan/pkg/gcsql" + "github.com/gochan-org/gochan/pkg/gcutil/testutil" "github.com/stretchr/testify/assert" lua "github.com/yuin/gopher-lua" luar "layeh.com/gopher-luar" @@ -38,7 +39,7 @@ return { ListenIP = system_critical_cfg.ListenIP, SiteSlogan = site_cfg.SiteSlog ) func initPluginTests() { - config.SetVersion("3.8.0") + config.SetVersion("4.0.2") initLua() } @@ -107,3 +108,14 @@ return joined, query_escaped, query_unescaped, err`) assert.Equal(t, errLV.Type(), lua.LTNil) ClosePlugins() } + +func TestLoadPlugin(t *testing.T) { + testutil.GoToGochanRoot(t) + initPluginTests() + assert.NoError(t, LoadPlugins([]string{"examples/plugins/uploadfilenameupper.lua"})) + assert.NoError(t, LoadPlugins(nil)) + assert.Error(t, LoadPlugins([]string{"not_a_file.lua"})) + assert.Error(t, LoadPlugins([]string{"invalid_ext.dll"})) + assert.ErrorContains(t, LoadPlugins([]string{"not_a_file.so"}), "realpath failed") + ClosePlugins() +} From 74f9e415b4e9352936137b3103bfcc0ab852b4a3 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Tue, 18 Feb 2025 19:50:41 -0800 Subject: [PATCH 115/122] Fix deepsource issues --- pkg/gcplugin/gcplugin.go | 1 - pkg/posting/formatting.go | 2 +- tools/get_js.py | 6 +++--- tools/selenium_testing/options.py | 2 +- tools/selenium_testing/runtests.py | 8 ++++---- tools/selenium_testing/tests/test_posting.py | 4 ++-- tools/selenium_testing/util/posting.py | 2 +- 7 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pkg/gcplugin/gcplugin.go b/pkg/gcplugin/gcplugin.go index b1e8d44c..04fcbe0b 100644 --- a/pkg/gcplugin/gcplugin.go +++ b/pkg/gcplugin/gcplugin.go @@ -31,7 +31,6 @@ import ( var ( lState *lua.LState - eventPlugins map[string][]*lua.LFunction ErrInvalidInitFunc = errors.New("invalid InitPlugin, expected function with 0 arguments and 1 return value (error type)") ) diff --git a/pkg/posting/formatting.go b/pkg/posting/formatting.go index eabf78d0..7f9812f5 100644 --- a/pkg/posting/formatting.go +++ b/pkg/posting/formatting.go @@ -156,7 +156,7 @@ func ApplyDiceRoll(p *gcsql.Post) (rollSum int, err error) { return 0, fmt.Errorf("dice roll too small") } for i := 0; i < numDice; i++ { - rollSum += rand.Intn(dieSize) + 1 + rollSum += rand.Intn(dieSize) + 1 // skipcq: GSC-G404 switch roll[3] { case "+": mod, err := strconv.Atoi(roll[4]) diff --git a/tools/get_js.py b/tools/get_js.py index 7971c403..11ca6d5c 100755 --- a/tools/get_js.py +++ b/tools/get_js.py @@ -11,8 +11,8 @@ from os import path import sys VERSION = "v3.10.2" -DIR = "gochan-{}_linux".format(VERSION) -DOWNLOAD_URL = "https://github.com/gochan-org/gochan/releases/download/{}/{}.tar.gz".format(VERSION, DIR) +DIR = f"gochan-{VERSION}_linux" +DOWNLOAD_URL = f"https://github.com/gochan-org/gochan/releases/download/{VERSION}/{DIR}.tar.gz" JS_DIR = path.join(DIR, "html/js/") if __name__ == "__main__": @@ -20,7 +20,7 @@ if __name__ == "__main__": if len(sys.argv) == 2: match sys.argv[1]: case "-h" | "--help": - print("usage: {} [path/to/out/js/]".format(sys.argv[0])) + print(f"usage: {sys.argv[0]} [path/to/out/js/]") sys.exit(0) case _: out_dir = sys.argv[1] diff --git a/tools/selenium_testing/options.py b/tools/selenium_testing/options.py index 35a14687..7a64fe4f 100644 --- a/tools/selenium_testing/options.py +++ b/tools/selenium_testing/options.py @@ -132,7 +132,7 @@ class TestingOptions: case ""|None: raise ValueError("browser argument is required") case _: - raise ValueError("Unrecognized browser argument %s" % browser) + raise ValueError(f"Unrecognized browser argument {browser}") def boards_json(self) -> dict[str, object]: diff --git a/tools/selenium_testing/runtests.py b/tools/selenium_testing/runtests.py index 7513a369..ed451beb 100755 --- a/tools/selenium_testing/runtests.py +++ b/tools/selenium_testing/runtests.py @@ -21,7 +21,7 @@ def start_tests(dict_options:dict[str,object]=None): options = TestingOptions.from_dict(dict_options) set_active_options(options) single_test = dict_options.get("single_test", "") - print("Using browser %s (headless: %s) on site %s" % (options.browser, options.headless, options.site)) + print(f"Using browser {options.browser} (headless: {options.headless}) on site {options.site}") if single_test == "": suite = unittest.suite.TestSuite() SeleniumTestCase.add(suite, options, TestPosting) @@ -69,7 +69,7 @@ def setup_selenium_args(parser:ArgumentParser): parser.add_argument("--keep-open", action="store_true", help="If set, the browser windows will not automatically close after the tests are complete") parser.add_argument("--site", default=default_site, - help=("Sets the site to be used for testing, defaults to %s" % default_site)) + help=f"Sets the site to be used for testing, defaults to {default_site}") parser.add_argument("--board1", default=default_board1, help="Sets the main board to be used for testing. It must already be created or tests that use it will fail") parser.add_argument("--board2", default=default_board2, @@ -102,14 +102,14 @@ def setup_selenium_args(parser:ArgumentParser): help="Sets the password to be used when logging in as a janitor. Janitor tests will fail if this does not exist") parser.add_argument("--single-test", default="", help="If specified, only the test method with this name will be run") - + return parser.parse_args() if __name__ == "__main__": parser = ArgumentParser(description="Browser testing via Selenium") args = setup_selenium_args(parser) - + try: start_tests(args.__dict__) except KeyboardInterrupt: diff --git a/tools/selenium_testing/tests/test_posting.py b/tools/selenium_testing/tests/test_posting.py index 73aad076..f56d875e 100644 --- a/tools/selenium_testing/tests/test_posting.py +++ b/tools/selenium_testing/tests/test_posting.py @@ -63,11 +63,11 @@ class TestPosting(SeleniumTestCase): cur_url = self.driver.current_url threadID = threadRE.findall(cur_url)[0][1] - self.driver.find_element(by=By.CSS_SELECTOR, value=("input#check%s"%threadID)).click() + self.driver.find_element(by=By.CSS_SELECTOR, value=f"input#check{threadID}").click() cur_url = self.driver.current_url self.driver.find_element(by=By.CSS_SELECTOR, value="input[name=move_btn]").click() # wait for response to move_btn - WebDriverWait(self.driver, 10).until(EC.title_contains("Move thread #%s" % threadID)) + WebDriverWait(self.driver, 10).until(EC.title_contains(f"Move thread #{threadID}")) self.driver.find_element(by=By.CSS_SELECTOR, value="input[type=submit]").click() # wait for response to move request (domove=1) diff --git a/tools/selenium_testing/util/posting.py b/tools/selenium_testing/util/posting.py index f6dbc99a..1dc03f00 100644 --- a/tools/selenium_testing/util/posting.py +++ b/tools/selenium_testing/util/posting.py @@ -52,7 +52,7 @@ def delete_post(options: TestingOptions, postID:int, password:str): qr_buttons = options.driver.find_element(by=By.ID, value="qr-buttons") if qr_buttons.is_displayed(): qr_buttons.find_element(by=By.LINK_TEXT, value="X").click() - options.driver.find_element(by=By.CSS_SELECTOR, value=("input#check%s"%postID)).click() + options.driver.find_element(by=By.CSS_SELECTOR, value=f"input#check{postID}").click() if password != "": delPasswordInput = options.driver.find_element( by=By.CSS_SELECTOR, From c842ca53f593b227b90995dfa2da056903153930 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Wed, 19 Feb 2025 03:57:55 +0000 Subject: [PATCH 116/122] refactor: unused parameter should be replaced by underscore Unused parameters in functions or methods should be replaced with `_` (underscore) or removed. --- cmd/gochan/main.go | 2 +- pkg/events/events_test.go | 6 +++--- pkg/gcsql/filterhandlers.go | 32 ++++++++++++++++---------------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cmd/gochan/main.go b/cmd/gochan/main.go index 10f44fb1..f556c630 100644 --- a/cmd/gochan/main.go +++ b/cmd/gochan/main.go @@ -78,7 +78,7 @@ func main() { gcutil.LogFatal().Err(err).Msg("Failed to initialize the database") } events.TriggerEvent("db-initialized") - events.RegisterEvent([]string{"db-views-reset"}, func(trigger string, i ...any) error { + events.RegisterEvent([]string{"db-views-reset"}, func(_ string, _ ...any) error { gcutil.LogInfo().Msg("SQL views reset") return nil }) diff --git a/pkg/events/events_test.go b/pkg/events/events_test.go index 0673d656..184b7003 100644 --- a/pkg/events/events_test.go +++ b/pkg/events/events_test.go @@ -7,7 +7,7 @@ import ( ) func TestPanicRecover(t *testing.T) { - RegisterEvent([]string{"TestPanicRecoverEvt"}, func(tr string, i ...any) error { + RegisterEvent([]string{"TestPanicRecoverEvt"}, func(_ string, i ...any) error { t.Log("Testing panic recover") t.Log(i[0]) return nil @@ -21,7 +21,7 @@ func TestPanicRecover(t *testing.T) { } func TestEventEditValue(t *testing.T) { - RegisterEvent([]string{"TestEventEditValue"}, func(tr string, i ...any) error { + RegisterEvent([]string{"TestEventEditValue"}, func(_ string, i ...any) error { p := i[0].(*int) *p += 1 return nil @@ -35,7 +35,7 @@ func TestEventEditValue(t *testing.T) { func TestMultipleEventTriggers(t *testing.T) { triggered := map[string]bool{} - RegisterEvent([]string{"a", "b"}, func(tr string, i ...any) error { + RegisterEvent([]string{"a", "b"}, func(tr string, _ ...any) error { triggered[tr] = true return nil }) diff --git a/pkg/gcsql/filterhandlers.go b/pkg/gcsql/filterhandlers.go index a619f31e..ff35b2e2 100644 --- a/pkg/gcsql/filterhandlers.go +++ b/pkg/gcsql/filterhandlers.go @@ -118,87 +118,87 @@ func init() { filterFieldHandlers = map[string]FilterConditionHandler{ "name": &conditionHandler{ fieldType: StringField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { return matchString(fc, p.Name) }, }, "trip": &conditionHandler{ fieldType: StringField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { return matchString(fc, p.Tripcode) }, }, "email": &conditionHandler{ fieldType: StringField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { return matchString(fc, p.Email) }, }, "subject": &conditionHandler{ fieldType: StringField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { return matchString(fc, p.Subject) }, }, "body": &conditionHandler{ fieldType: StringField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { return matchString(fc, p.MessageRaw) }, }, "firsttimeboard": &conditionHandler{ fieldType: BooleanField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { return firstPost(p, false) }, }, "notfirsttimeboard": &conditionHandler{ fieldType: BooleanField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { first, err := firstPost(p, false) return !first, err }, }, "firsttimesite": &conditionHandler{ fieldType: BooleanField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { return firstPost(p, true) }, }, "notfirsttimesite": &conditionHandler{ fieldType: BooleanField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { first, err := firstPost(p, true) return !first, err }, }, "isop": &conditionHandler{ fieldType: BooleanField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { return p.IsTopPost, nil }, }, "notop": &conditionHandler{ fieldType: BooleanField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) { return !p.IsTopPost, nil }, }, "hasfile": &conditionHandler{ fieldType: BooleanField, - matchFunc: func(r *http.Request, p *Post, u *Upload, fc *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, _ *Post, u *Upload, _ *FilterCondition) (bool, error) { return u != nil, nil }, }, "nofile": &conditionHandler{ fieldType: BooleanField, - matchFunc: func(r *http.Request, p *Post, u *Upload, fc *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, _ *Post, u *Upload, _ *FilterCondition) (bool, error) { return u == nil, nil }, }, "filename": &conditionHandler{ fieldType: StringField, - matchFunc: func(r *http.Request, p *Post, u *Upload, fc *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, _ *Post, u *Upload, fc *FilterCondition) (bool, error) { if u == nil { return false, nil } @@ -207,7 +207,7 @@ func init() { }, "checksum": &conditionHandler{ fieldType: StringField, - matchFunc: func(r *http.Request, p *Post, u *Upload, fc *FilterCondition) (bool, error) { + matchFunc: func(_ *http.Request, _ *Post, u *Upload, fc *FilterCondition) (bool, error) { if u == nil { return false, nil } @@ -216,7 +216,7 @@ func init() { }, "useragent": &conditionHandler{ fieldType: StringField, - matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) { + matchFunc: func(r *http.Request, _ *Post, _ *Upload, fc *FilterCondition) (bool, error) { return matchString(fc, r.UserAgent()) }, }, From 39fdc0566fd4ccaa7b04997d0334ba9252613e5a Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Wed, 19 Feb 2025 04:05:57 +0000 Subject: [PATCH 117/122] refactor: fix unused method receiver Methods with unused receivers can be a symptom of unfinished refactoring or a bug. To keep the same method signature, omit the receiver name or '_' as it is unused. --- cmd/gochan-migration/internal/gcupdate/gcupdate.go | 2 +- cmd/gochan-migration/internal/pre2021/bans.go | 2 +- cmd/gochan-migration/internal/pre2021/posts.go | 2 +- pkg/server/serverutil/util_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/gochan-migration/internal/gcupdate/gcupdate.go b/cmd/gochan-migration/internal/gcupdate/gcupdate.go index faaf69b8..5b7a9209 100644 --- a/cmd/gochan-migration/internal/gcupdate/gcupdate.go +++ b/cmd/gochan-migration/internal/gcupdate/gcupdate.go @@ -22,7 +22,7 @@ type GCDatabaseUpdater struct { } // IsMigratingInPlace implements common.DBMigrator. -func (dbu *GCDatabaseUpdater) IsMigratingInPlace() bool { +func (*GCDatabaseUpdater) IsMigratingInPlace() bool { return true } diff --git a/cmd/gochan-migration/internal/pre2021/bans.go b/cmd/gochan-migration/internal/pre2021/bans.go index 5a0bfd10..19027380 100644 --- a/cmd/gochan-migration/internal/pre2021/bans.go +++ b/cmd/gochan-migration/internal/pre2021/bans.go @@ -66,7 +66,7 @@ func (m *Pre2021Migrator) migrateBansInPlace() error { return m.migrateBansToNewDB() } -func (m *Pre2021Migrator) migrateBan(tx *sql.Tx, ban *migrationBan, boardID *int, errEv *zerolog.Event) error { +func (*Pre2021Migrator) migrateBan(tx *sql.Tx, ban *migrationBan, boardID *int, errEv *zerolog.Event) error { migratedBan := &gcsql.IPBan{ BoardID: boardID, RangeStart: ban.ip, diff --git a/cmd/gochan-migration/internal/pre2021/posts.go b/cmd/gochan-migration/internal/pre2021/posts.go index 3fa474fc..a773fb35 100644 --- a/cmd/gochan-migration/internal/pre2021/posts.go +++ b/cmd/gochan-migration/internal/pre2021/posts.go @@ -32,7 +32,7 @@ type migrationPost struct { oldParentID int } -func (m *Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *zerolog.Event) error { +func (*Pre2021Migrator) migratePost(tx *sql.Tx, post *migrationPost, errEv *zerolog.Event) error { var err error opts := &gcsql.RequestOptions{Tx: tx} if post.oldParentID == 0 { diff --git a/pkg/server/serverutil/util_test.go b/pkg/server/serverutil/util_test.go index 903a04c2..bd9879d9 100644 --- a/pkg/server/serverutil/util_test.go +++ b/pkg/server/serverutil/util_test.go @@ -62,7 +62,7 @@ type testResponseWriter struct { func (w *testResponseWriter) Header() http.Header { return w.header } -func (w *testResponseWriter) Write([]byte) (int, error) { +func (*testResponseWriter) Write([]byte) (int, error) { return 0, nil } func (w *testResponseWriter) WriteHeader(s int) { From 0244811a7fcc8e2400801b39447161522771c06e Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Tue, 18 Feb 2025 20:33:33 -0800 Subject: [PATCH 118/122] Fix more deepsource issues --- .../internal/pre2021/pre2021_test.go | 2 +- pkg/manage/actionsModPerm.go | 2 +- pkg/posting/formatting.go | 24 +++++++++---------- pkg/posting/formatting_test.go | 19 +++++++++++++++ pkg/server/serverutil/antispam_test.go | 2 +- pkg/server/serverutil/util_test.go | 4 ++-- tools/get_js.py | 2 +- .../tests/test_staff_permissions.py | 2 +- 8 files changed, 38 insertions(+), 19 deletions(-) diff --git a/cmd/gochan-migration/internal/pre2021/pre2021_test.go b/cmd/gochan-migration/internal/pre2021/pre2021_test.go index d424eb53..5afa2afa 100644 --- a/cmd/gochan-migration/internal/pre2021/pre2021_test.go +++ b/cmd/gochan-migration/internal/pre2021/pre2021_test.go @@ -38,7 +38,7 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20 } defer oldDbFile.Close() - newDbFile, err := os.OpenFile(migratedDBHost, os.O_CREATE|os.O_WRONLY, 0644) + newDbFile, err := os.OpenFile(migratedDBHost, os.O_CREATE|os.O_WRONLY, 0600) if !assert.NoError(t, err) { t.FailNow() } diff --git a/pkg/manage/actionsModPerm.go b/pkg/manage/actionsModPerm.go index 36047d1a..52cffd05 100644 --- a/pkg/manage/actionsModPerm.go +++ b/pkg/manage/actionsModPerm.go @@ -219,7 +219,7 @@ func filterHitsCallback(writer http.ResponseWriter, request *http.Request, staff errEv.Err(err).Caller().RawJSON("postData", []byte(hit.PostData)).Msg("Unable to marshal un-minified post data") return nil, err } - hitsJSON = append(hitsJSON, template.HTML(strings.ReplaceAll(jsonBuf.String(), "\n", "
"))) + hitsJSON = append(hitsJSON, template.HTML(strings.ReplaceAll(jsonBuf.String(), "\n", "
"))) // skipcq: GSC-G203 } var buf bytes.Buffer if err = serverutil.MinifyTemplate(gctemplates.ManageFilterHits, map[string]any{ diff --git a/pkg/posting/formatting.go b/pkg/posting/formatting.go index 7f9812f5..46eab445 100644 --- a/pkg/posting/formatting.go +++ b/pkg/posting/formatting.go @@ -18,7 +18,7 @@ var ( msgfmtr MessageFormatter urlRE = regexp.MustCompile(`https?://(\S+)`) unsetBBcodeTags = []string{"center", "color", "img", "quote", "size"} - diceRoller = regexp.MustCompile(`(?i)\[(\d*)d(\d+)(?:([+-])(\d+))?\]`) + diceRoller = regexp.MustCompile(`(?i)(\S*)\[(\d*)d(\d+)(?:([+-])(\d+))?\](\S*)`) ) // InitPosting prepares the formatter and the temp post pruner @@ -142,13 +142,13 @@ func ApplyDiceRoll(p *gcsql.Post) (rollSum int, err error) { continue } numDice := 1 - if roll[1] != "" { - numDice, err = strconv.Atoi(roll[1]) + if roll[2] != "" { + numDice, err = strconv.Atoi(roll[2]) if err != nil { return 0, err } } - dieSize, err := strconv.Atoi(roll[2]) + dieSize, err := strconv.Atoi(roll[3]) if err != nil { return 0, err } @@ -157,27 +157,27 @@ func ApplyDiceRoll(p *gcsql.Post) (rollSum int, err error) { } for i := 0; i < numDice; i++ { rollSum += rand.Intn(dieSize) + 1 // skipcq: GSC-G404 - switch roll[3] { + switch roll[4] { case "+": - mod, err := strconv.Atoi(roll[4]) + mod, err := strconv.Atoi(roll[5]) if err != nil { return 0, err } rollSum += mod case "-": - mod, err := strconv.Atoi(roll[4]) + mod, err := strconv.Atoi(roll[5]) if err != nil { return 0, err } rollSum -= mod } } - words[w] = fmt.Sprintf(`%dd%d`, numDice, dieSize) - if roll[3] != "" { - words[w] += roll[3] + roll[4] + words[w] = fmt.Sprintf(`%s%dd%d`, roll[1], numDice, dieSize) + if roll[4] != "" { + words[w] += roll[4] + roll[5] } - words[w] += fmt.Sprintf(" = %d", rollSum) + words[w] += fmt.Sprintf(" = %d%s", rollSum, roll[6]) } - p.Message = template.HTML(strings.Join(words, " ")) + p.Message = template.HTML(strings.Join(words, " ")) // skipcq: GSC-G203 return } diff --git a/pkg/posting/formatting_test.go b/pkg/posting/formatting_test.go index 9eedeb38..2013c561 100644 --- a/pkg/posting/formatting_test.go +++ b/pkg/posting/formatting_test.go @@ -74,6 +74,25 @@ var ( expectMin: 1, expectMax: 8, }, + { + desc: "before[1d6]after, no space", + post: gcsql.Post{ + MessageRaw: "before[1d6]after", + }, + matcher: regexp.MustCompile(`before1d6 = \dafter`), + expectMin: 1, + expectMax: 6, + }, + { + desc: "before [1d6] after, no space (test for injection)", + post: gcsql.Post{ + MessageRaw: `[1d6]`, + }, + expectError: false, + matcher: regexp.MustCompile(`<script>alert\("lol"\)</script>1d6 = \d<script>alert\("lmao"\)</script>`), + expectMin: 1, + expectMax: 6, + }, } ) diff --git a/pkg/server/serverutil/antispam_test.go b/pkg/server/serverutil/antispam_test.go index 6c06984a..52172b12 100644 --- a/pkg/server/serverutil/antispam_test.go +++ b/pkg/server/serverutil/antispam_test.go @@ -52,7 +52,7 @@ type checkRefererTestCase struct { func TestCheckReferer(t *testing.T) { config.SetVersion("4.0.0") systemCriticalConfig := config.GetSystemCriticalConfig() - req, err := http.NewRequest("GET", "http://gochan.org", nil) + req, err := http.NewRequest("GET", "https://gochan.org", nil) if !assert.NoError(t, err) { t.FailNow() } diff --git a/pkg/server/serverutil/util_test.go b/pkg/server/serverutil/util_test.go index bd9879d9..e86ebd01 100644 --- a/pkg/server/serverutil/util_test.go +++ b/pkg/server/serverutil/util_test.go @@ -36,7 +36,7 @@ type isRequestingJSONTestCase struct { } func TestIsRequestingJSON(t *testing.T) { - req, _ := http.NewRequest("GET", "http://localhost:8080", nil) + req, _ := http.NewRequest("GET", "https://localhost:8080", nil) assert.False(t, IsRequestingJSON(req)) for _, tc := range isRequestingJSONTestCases { t.Run("GET "+tc.val, func(t *testing.T) { @@ -70,7 +70,7 @@ func (w *testResponseWriter) WriteHeader(s int) { } func TestDeleteCookie(t *testing.T) { - req, _ := http.NewRequest("GET", "http://localhost:8080", nil) + req, _ := http.NewRequest("GET", "https://localhost:8080", nil) writer := testResponseWriter{ header: make(http.Header), } diff --git a/tools/get_js.py b/tools/get_js.py index 11ca6d5c..8fa8557b 100755 --- a/tools/get_js.py +++ b/tools/get_js.py @@ -25,7 +25,7 @@ if __name__ == "__main__": case _: out_dir = sys.argv[1] - with urlopen(DOWNLOAD_URL) as response: + with urlopen(DOWNLOAD_URL) as response: # skipcq: BAN-B310 data = response.read() tar_bytes = gzip.decompress(data) buf = io.BytesIO(tar_bytes) diff --git a/tools/selenium_testing/tests/test_staff_permissions.py b/tools/selenium_testing/tests/test_staff_permissions.py index 521e5955..e36dea69 100644 --- a/tools/selenium_testing/tests/test_staff_permissions.py +++ b/tools/selenium_testing/tests/test_staff_permissions.py @@ -27,7 +27,7 @@ class TestStaffPermissions(SeleniumTestCase): req = Request(urljoin(options.site, "manage/actions")) # modern browsers add pretty printing to JSON so we need to pass the session cookie to a request to get the raw action list data req.add_header("Cookie", f"sessiondata={cookie}") - with urlopen(req) as resp: + with urlopen(req) as resp: # skipcq: BAN-B310 global actions actions = json.load(resp) From 282f32c88d3dc3906e2ab0a9acece386ae4a895d Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Tue, 18 Feb 2025 20:49:43 -0800 Subject: [PATCH 119/122] Bump build version for v4.1 release --- build.py | 2 +- frontend/package-lock.json | 4 ++-- frontend/package.json | 2 +- html/error/404.html | 2 +- html/error/500.html | 2 +- html/error/502.html | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build.py b/build.py index e2c6feb6..92318b2f 100755 --- a/build.py +++ b/build.py @@ -39,7 +39,7 @@ release_files = ( "README.md", ) -GOCHAN_VERSION = "4.0.2" +GOCHAN_VERSION = "4.1.0" DATABASE_VERSION = "4" # stored in DBNAME.DBPREFIXdatabase_version PATH_NOTHING = -1 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 40b8f3a3..1bc88fd2 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "gochan.js", - "version": "4.0.2", + "version": "4.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gochan.js", - "version": "4.0.2", + "version": "4.1.0", "license": "BSD-2-Clause", "dependencies": { "jquery": "^3.7.1", diff --git a/frontend/package.json b/frontend/package.json index 79eadec7..160f51ec 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "gochan.js", - "version": "4.0.2", + "version": "4.1.0", "description": "", "main": "./ts/main.ts", "private": true, diff --git a/html/error/404.html b/html/error/404.html index 602780fd..efe84544 100755 --- a/html/error/404.html +++ b/html/error/404.html @@ -7,6 +7,6 @@

404: File not found

lol 404

The requested file could not be found on this server.

-
Site powered by Gochan v4.0.2 +
Site powered by Gochan v4.1.0 \ No newline at end of file diff --git a/html/error/500.html b/html/error/500.html index 8fe12c28..fd1643cb 100755 --- a/html/error/500.html +++ b/html/error/500.html @@ -7,6 +7,6 @@

Error 500: Internal Server error

server burning

The server encountered an error while trying to serve the page, and we apologize for the inconvenience. The system administrator will try to fix things as soon they get around to it, whenever that is. Hopefully soon.

-
Site powered by Gochan v4.0.2 +
Site powered by Gochan v4.1.0 \ No newline at end of file diff --git a/html/error/502.html b/html/error/502.html index e5644dd5..d6013c5e 100644 --- a/html/error/502.html +++ b/html/error/502.html @@ -7,6 +7,6 @@

Error 502: Bad gateway

server burning

The server encountered an error while trying to serve the page, and we apologize for the inconvenience. The system administrator will try to fix things as soon they get around to it, whenever that is. Hopefully soon.

-
Site powered by Gochan v4.0.2 +
Site powered by Gochan v4.1.0 \ No newline at end of file From 0ec52bfdf1b99412e13b8d93fbc6cef75369ec17 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Wed, 19 Feb 2025 17:10:57 -0800 Subject: [PATCH 120/122] Improve dice roll parsing --- pkg/posting/formatting.go | 85 +++++++++++++++------------------ pkg/posting/formatting_test.go | 87 ++++++++++++++++++++++++---------- pkg/posting/post.go | 2 +- 3 files changed, 101 insertions(+), 73 deletions(-) diff --git a/pkg/posting/formatting.go b/pkg/posting/formatting.go index 46eab445..ac704c65 100644 --- a/pkg/posting/formatting.go +++ b/pkg/posting/formatting.go @@ -1,6 +1,7 @@ package posting import ( + "errors" "fmt" "html/template" "math/rand" @@ -18,7 +19,7 @@ var ( msgfmtr MessageFormatter urlRE = regexp.MustCompile(`https?://(\S+)`) unsetBBcodeTags = []string{"center", "color", "img", "quote", "size"} - diceRoller = regexp.MustCompile(`(?i)(\S*)\[(\d*)d(\d+)(?:([+-])(\d+))?\](\S*)`) + diceRollRE = regexp.MustCompile(`\[(\d*)d(\d+)(?:([+-])(\d+))?\]`) ) // InitPosting prepares the formatter and the temp post pruner @@ -134,50 +135,42 @@ func FormatMessage(message string, boardDir string) (template.HTML, error) { return template.HTML(strings.Join(postLines, "
")), nil // skipcq: GSC-G203 } -func ApplyDiceRoll(p *gcsql.Post) (rollSum int, err error) { - words := strings.Split(string(p.Message), " ") - for w, word := range words { - roll := diceRoller.FindStringSubmatch(word) - if len(roll) == 0 { - continue - } - numDice := 1 - if roll[2] != "" { - numDice, err = strconv.Atoi(roll[2]) - if err != nil { - return 0, err - } - } - dieSize, err := strconv.Atoi(roll[3]) - if err != nil { - return 0, err - } - if numDice < 1 || dieSize < 1 { - return 0, fmt.Errorf("dice roll too small") - } - for i := 0; i < numDice; i++ { - rollSum += rand.Intn(dieSize) + 1 // skipcq: GSC-G404 - switch roll[4] { - case "+": - mod, err := strconv.Atoi(roll[5]) - if err != nil { - return 0, err - } - rollSum += mod - case "-": - mod, err := strconv.Atoi(roll[5]) - if err != nil { - return 0, err - } - rollSum -= mod - } - } - words[w] = fmt.Sprintf(`%s%dd%d`, roll[1], numDice, dieSize) - if roll[4] != "" { - words[w] += roll[4] + roll[5] - } - words[w] += fmt.Sprintf(" = %d%s", rollSum, roll[6]) +func diceRoller(numDice int, diceSides int, modifier int) int { + rollSum := 0 + for i := 0; i < numDice; i++ { + rollSum += rand.Intn(diceSides) + 1 // skipcq: GSC-G404 } - p.Message = template.HTML(strings.Join(words, " ")) // skipcq: GSC-G203 - return + return rollSum + modifier +} + +func ApplyDiceRoll(p *gcsql.Post) error { + var err error + result := diceRollRE.ReplaceAllStringFunc(string(p.Message), func(roll string) string { + rollMatch := diceRollRE.FindStringSubmatch(roll) + numDice := 1 + if rollMatch[1] != "" { + numDice, _ = strconv.Atoi(rollMatch[1]) + } + if numDice < 1 { + err = errors.New("number of dice must be at least 1") + return roll + } + dieSize, _ := strconv.Atoi(rollMatch[2]) + if dieSize <= 1 { + err = errors.New("die size must be greater than 1") + return roll + } + modifierIsNegative := rollMatch[3] == "-" + modifier, _ := strconv.Atoi(rollMatch[4]) + if modifierIsNegative { + modifier = -modifier + } + rollSum := diceRoller(numDice, dieSize, modifier) + return fmt.Sprintf(`%dd%d%s%s = %d`, numDice, dieSize, rollMatch[3], rollMatch[4], rollSum) + }) + if err != nil { + return err + } + p.Message = template.HTML(result) // skipcq: GSC-G203 + return nil } diff --git a/pkg/posting/formatting_test.go b/pkg/posting/formatting_test.go index 2013c561..d6c79039 100644 --- a/pkg/posting/formatting_test.go +++ b/pkg/posting/formatting_test.go @@ -39,31 +39,31 @@ end)` var ( diceTestCases = []diceRollerTestCase{ { - desc: "[1d6]", + desc: "[2d6]", post: gcsql.Post{ - MessageRaw: "before [1d6] after", + MessageRaw: "[2d6]", }, - matcher: regexp.MustCompile(`before 1d6 = \d after`), - expectMin: 1, - expectMax: 6, - }, - { - desc: "[1d6+1]", - post: gcsql.Post{ - MessageRaw: "before [1d6+1] after", - }, - matcher: regexp.MustCompile(`before 1d6\+1 = \d after`), + matcher: regexp.MustCompile(`2d6 = \d{1,2}`), expectMin: 2, - expectMax: 7, + expectMax: 12, }, { - desc: "[1d6-1]", + desc: "[2d6+1]", post: gcsql.Post{ - MessageRaw: "before [1d6-1] after", + MessageRaw: "[2d6+1]", }, - matcher: regexp.MustCompile(`before 1d6-1 = \d after`), - expectMin: 0, - expectMax: 5, + matcher: regexp.MustCompile(`2d6\+1 = \d{1,2}`), + expectMin: 3, + expectMax: 13, + }, + { + desc: "[2d6-1]", + post: gcsql.Post{ + MessageRaw: "[2d6-1]", + }, + matcher: regexp.MustCompile(`2d6-1 = \d{1,2}`), + expectMin: 1, + expectMax: 11, }, { desc: "[d8]", @@ -88,10 +88,48 @@ var ( post: gcsql.Post{ MessageRaw: `[1d6]`, }, - expectError: false, - matcher: regexp.MustCompile(`<script>alert\("lol"\)</script>1d6 = \d<script>alert\("lmao"\)</script>`), - expectMin: 1, - expectMax: 6, + matcher: regexp.MustCompile(`<script>alert\("lol"\)</script>1d6 = \d<script>alert\("lmao"\)</script>`), + expectMin: 1, + expectMax: 6, + }, + { + desc: "two dice rolls, no space", + post: gcsql.Post{ + MessageRaw: "[d6][2d6]", + }, + matcher: regexp.MustCompile(`1d6 = \d2d6 = \d{1,2}`), + expectMin: 0, + expectMax: 7, + }, + { + desc: "multiple dice rolls, no space", + post: gcsql.Post{ + MessageRaw: "[d6][2d20-2][3d8+1]", + }, + matcher: regexp.MustCompile(`1d6 = \d2d20-2 = \d{1,2}3d8\+1 = \d{1,2}`), + expectMin: 0, + expectMax: 38, + }, + { + desc: "invalid number of dice", + post: gcsql.Post{ + MessageRaw: "[0d6]", + }, + expectError: true, + }, + { + desc: "invalid die size", + post: gcsql.Post{ + MessageRaw: "[1d0]", + }, + expectError: true, + }, + { + desc: "invalid modifier", + post: gcsql.Post{ + MessageRaw: "[1d6+]", + }, + matcher: regexp.MustCompile(`\[1d6\+\]`), }, } ) @@ -148,15 +186,12 @@ func diceRollRunner(t *testing.T, tC *diceRollerTestCase) { var err error tC.post.Message, err = FormatMessage(tC.post.MessageRaw, "") assert.NoError(t, err) - result, err := ApplyDiceRoll(&tC.post) + err = ApplyDiceRoll(&tC.post) if tC.expectError { assert.Error(t, err) - assert.Equal(t, 0, result) } else { assert.NoError(t, err) assert.Regexp(t, tC.matcher, tC.post.Message) - assert.GreaterOrEqual(t, result, tC.expectMin) - assert.LessOrEqual(t, result, tC.expectMax) } if t.Failed() { t.FailNow() diff --git a/pkg/posting/post.go b/pkg/posting/post.go index 3e4999d0..94596508 100644 --- a/pkg/posting/post.go +++ b/pkg/posting/post.go @@ -226,7 +226,7 @@ func doFormatting(post *gcsql.Post, board *gcsql.Board, request *http.Request, e errEv.Err(err).Caller().Msg("Unable to format message") return errors.New("unable to format message") } - if _, err = ApplyDiceRoll(post); err != nil { + if err = ApplyDiceRoll(post); err != nil { errEv.Err(err).Caller().Msg("Error applying dice roll") return err } From 41fa1f6280c2981924ffb25c4f117b524dcddda3 Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sat, 22 Feb 2025 10:41:54 -0800 Subject: [PATCH 121/122] Migrate SCSS, replace usage of @import with @use https://sass-lang.com/documentation/breaking-changes/import/ --- frontend/sass/_yotsubacommon.scss | 4 +-- frontend/sass/bunkerchan.scss | 30 +++++++++--------- frontend/sass/bunkerchan/_front.scss | 6 ++-- frontend/sass/bunkerchan/_img.scss | 14 ++++----- frontend/sass/burichan.scss | 26 ++++++++-------- frontend/sass/burichan/_img.scss | 15 ++++----- frontend/sass/burichan/_manage.scss | 8 ++--- frontend/sass/clear.scss | 36 +++++++++++----------- frontend/sass/dark.scss | 38 +++++++++++------------ frontend/sass/darkbunker.scss | 30 +++++++++--------- frontend/sass/darkbunker/_img.scss | 16 +++++----- frontend/sass/global.scss | 16 +++++----- frontend/sass/global/_img.scss | 2 +- frontend/sass/global/_lightbox.scss | 21 +++++-------- frontend/sass/global/_qr.scss | 6 ++-- frontend/sass/photon.scss | 20 ++++++------ frontend/sass/photon/_img.scss | 20 ++++++------ frontend/sass/pipes.scss | 46 ++++++++++++++-------------- frontend/sass/pipes/_front.scss | 18 +++++------ frontend/sass/pipes/_img.scss | 16 +++++----- frontend/sass/pipes/_manage.scss | 8 ++--- frontend/sass/win9x.scss | 42 ++++++++++++------------- frontend/sass/yotsuba.scss | 28 ++++++++--------- frontend/sass/yotsubab.scss | 26 ++++++++-------- html/css/global.css | 20 ------------ 25 files changed, 244 insertions(+), 268 deletions(-) diff --git a/frontend/sass/_yotsubacommon.scss b/frontend/sass/_yotsubacommon.scss index b4ed4169..702f0a58 100644 --- a/frontend/sass/_yotsubacommon.scss +++ b/frontend/sass/_yotsubacommon.scss @@ -1,4 +1,4 @@ -@import 'util'; +@use 'util'; @mixin yotsuba( $fadepath, @@ -106,5 +106,5 @@ border: 1px dashed #222; } - @include upload-box(#aaa, #444, #666); + @include util.upload-box(#aaa, #444, #666); } \ No newline at end of file diff --git a/frontend/sass/bunkerchan.scss b/frontend/sass/bunkerchan.scss index 73dde2d3..b606eebe 100644 --- a/frontend/sass/bunkerchan.scss +++ b/frontend/sass/bunkerchan.scss @@ -1,12 +1,12 @@ -@import 'bunkerchan/colors'; -@import 'bunkerchan/front'; -@import 'bunkerchan/img'; -@import 'util'; +@use 'bunkerchan/colors'; +@use 'bunkerchan/front'; +@use 'bunkerchan/img'; +@use 'util'; body { - background: $bgcol; - color: $color; - font-family: $font-family; + background: colors.$bgcol; + color: colors.$color; + font-family: colors.$font-family; font-size: 70%; } @@ -19,30 +19,30 @@ hr { } a, a:visited { - color: $topborder; + color: colors.$topborder; text-decoration: none; } div#staff, select.post-actions { - background: $topbarbg; - border: 1px solid $topborder; + background: colors.$topbarbg; + border: 1px solid colors.$topborder; } div#topbar, div#topbar a, div#topbar a:visited, div.dropdown-menu { - background: $topbarbg; - border-bottom: 1px solid $topborder; + background: colors.$topbarbg; + border-bottom: 1px solid colors.$topborder; } div.dropdown-menu { - border-left: 1px solid $topborder; - border-right: 1px solid $topborder; + border-left: 1px solid colors.$topborder; + border-right: 1px solid colors.$topborder; } div.dropdown-menu a:hover, div.dropdown-menu a:active { - background: $bgcol; + background: colors.$bgcol; } table#pages, table#pages * { diff --git a/frontend/sass/bunkerchan/_front.scss b/frontend/sass/bunkerchan/_front.scss index 496027b5..269190b5 100644 --- a/frontend/sass/bunkerchan/_front.scss +++ b/frontend/sass/bunkerchan/_front.scss @@ -1,8 +1,8 @@ -@import 'colors'; +@use 'colors'; div.section-title-block { - background: $topbarbg; - border: 1px solid $topborder; + background: colors.$topbarbg; + border: 1px solid colors.$topborder; border-radius: 5px 5px 0px 0px; } diff --git a/frontend/sass/bunkerchan/_img.scss b/frontend/sass/bunkerchan/_img.scss index df840890..68326cce 100644 --- a/frontend/sass/bunkerchan/_img.scss +++ b/frontend/sass/bunkerchan/_img.scss @@ -1,15 +1,15 @@ -@import 'colors'; +@use 'colors'; %formstyle { - background: $inputbg; - border: 1px double $inputborder; - color: $color; + background: colors.$inputbg; + border: 1px double colors.$inputborder; + color: colors.$color; } %darkselect { - background: $inputbg2; + background: colors.$inputbg2; border-radius: 5px; - color: $color; + color: colors.$color; @extend %formstyle; } @@ -34,7 +34,7 @@ table#postbox-static, div#report-delbox, form#filterform { @extend %darkselect; } button, input[type=submit] { - background: $inputbg2; + background: colors.$inputbg2; } } diff --git a/frontend/sass/burichan.scss b/frontend/sass/burichan.scss index 38d47de0..2502f30b 100644 --- a/frontend/sass/burichan.scss +++ b/frontend/sass/burichan.scss @@ -1,17 +1,17 @@ -@import 'util'; -@import 'burichan/colors'; -@import 'burichan/img'; -@import 'burichan/front'; -@import 'burichan/manage'; +@use 'util'; +@use 'burichan/colors'; +@use 'burichan/img'; +@use 'burichan/front'; +@use 'burichan/manage'; body { - font: $font; - background: $bgcol; + font: colors.$font; + background: colors.$bgcol; margin: 8px; } a, a:visited { - color: $linkcol; + color: colors.$linkcol; text-decoration: none; } @@ -36,7 +36,7 @@ h3 { } h1, h2 { - font-family: $hfont-family; + font-family: colors.$hfont-family; } h1, h2, h3 { @@ -45,7 +45,7 @@ h1, h2, h3 { } div#topbar { - @include shadow-filter(3px 5px 6px $shadowcol); + @include util.shadow-filter(3px 5px 6px colors.$shadowcol); min-height: 1.5em; } @@ -55,13 +55,13 @@ a.topbar-item:visited, .dropdown-button, div.dropdown-menu { background: #080e5e; - color: $bgcol; + color: colors.$bgcol; } div.dropdown-menu { - @include shadow-filter(3px 5px 6px $shadowcol); + @include util.shadow-filter(3px 5px 6px colors.$shadowcol); a, h3 { - color: $bgcol; + color: colors.$bgcol; } } diff --git a/frontend/sass/burichan/_img.scss b/frontend/sass/burichan/_img.scss index 9a94bb94..36267fd1 100644 --- a/frontend/sass/burichan/_img.scss +++ b/frontend/sass/burichan/_img.scss @@ -1,14 +1,15 @@ -@import 'colors'; -@import '../global/colors'; +@use 'colors'; +@use '../global/colors' as global-colors; +@use "../util"; h1#board-title { font-family: serif; font-size: 2em; - color: $headercol; + color: global-colors.$headercol; } .postblock { - background:$postblock; + background:colors.$postblock; } div.file-info { @@ -19,7 +20,7 @@ div.file-info { span.postername { font-size: 1em; font-family: serif; - color: $namecol; + color: global-colors.$namecol; font-weight: 800; } @@ -34,11 +35,11 @@ select.post-actions, div.reply, div.postprev, div.inlinepostprev { - border: 1px solid $bordercol; + border: 1px solid colors.$bordercol; } .dice-roll { border: 1px dashed #222; } -@include upload-box(#aaa, #444, #666); \ No newline at end of file +@include util.upload-box(#aaa, #444, #666); \ No newline at end of file diff --git a/frontend/sass/burichan/_manage.scss b/frontend/sass/burichan/_manage.scss index 3f60aba1..114c563e 100644 --- a/frontend/sass/burichan/_manage.scss +++ b/frontend/sass/burichan/_manage.scss @@ -1,17 +1,17 @@ -@import 'colors'; -@import '../util'; +@use 'colors'; +@use '../util'; .loginbox input { height: 20%; } .manage-header { - background: $bgcol; + background: colors.$bgcol; border-radius: 8px; } table.mgmt-table { tr:first-of-type th { - background: $postblock; + background: colors.$postblock; } } \ No newline at end of file diff --git a/frontend/sass/clear.scss b/frontend/sass/clear.scss index 356b173f..1ba74583 100644 --- a/frontend/sass/clear.scss +++ b/frontend/sass/clear.scss @@ -1,51 +1,51 @@ -@import 'clear/colors'; -@import './util'; +@use 'clear/colors'; +@use 'util'; body { - background: $bgcol; + background: colors.$bgcol; font-family: monospace, sans-serif; font-size: 80%; } div#topbar { - background: $topbarbg; - border-bottom: 1px solid $topbarborder; + background: colors.$topbarbg; + border-bottom: 1px solid colors.$topbarborder; } a { - color: $linkcol; + color: colors.$linkcol; text-decoration: none; &:hover { - text-shadow: 0px 0px 5px $hrcol; + text-shadow: 0px 0px 5px colors.$hrcol; } } header { - color: $headercol; + color: colors.$headercol; } hr { border: none; - border-top: 1px solid $hrcol; + border-top: 1px solid colors.$hrcol; height: 0; } div#content { input, select, textarea { - border: 1px double $inputshadow; + border: 1px double colors.$inputshadow; border-radius: 5px; - background: $inputcol; + background: colors.$inputcol; color: #000; } input:active, select:active, textarea:active { - @include shadow-filter(0px 0px 5px $replyborder); + @include util.shadow-filter(0px 0px 5px colors.$replyborder); } input[type="button"], input[type="submit"], input[type="file"]::file-selector-button, input[type="file"]::webkit-file-upload-button { background: #A7A7A7; - border: 3px double $inputshadow; + border: 3px double colors.$inputshadow; border-radius: 5px; color: #000; } @@ -58,14 +58,14 @@ th.postblock, div.postprev, div.inlinepostprev, table.mgmt-table tr:first-of-type th { - background: $inputcol; - border: 1px solid $replyborder; + background: colors.$inputcol; + border: 1px solid colors.$replyborder; border-radius: 5px; } span.subject { font-weight: 800; - color: $subjectcol; + color: colors.$subjectcol; } table#pages, table#pages * { @@ -74,7 +74,7 @@ table#pages, table#pages * { div.section-title-block { background: #A7A7A7; - border-bottom: 1px solid $replyborder; + border-bottom: 1px solid colors.$replyborder; } -@include upload-box(#aaa, #444, #666); \ No newline at end of file +@include util.upload-box(#aaa, #444, #666); \ No newline at end of file diff --git a/frontend/sass/dark.scss b/frontend/sass/dark.scss index c28c2078..04ecf774 100644 --- a/frontend/sass/dark.scss +++ b/frontend/sass/dark.scss @@ -1,41 +1,41 @@ -@import 'dark/colors'; +@use 'dark/colors'; body { - background: $bgcol; - font-family: $font-family; - color: $color; + background: colors.$bgcol; + font-family: colors.$font-family; + color: colors.$color; // font-size: 80%; } a { text-decoration: none; - color: $linkcol; + color: colors.$linkcol; } div#topbar { - background: $topbarbg; - border-bottom: 1px solid $color; + background: colors.$topbarbg; + border-bottom: 1px solid colors.$color; a { text-shadow: black 1px 1px 1px, black -1px -1px 1px, black -1px 1px 1px, black 1px -1px 1px; } } header { - color: $headercol; + color: colors.$headercol; } div#content { input:not(div#qrbuttons input), textarea, select { - background: $inputbg; - border: 1px solid $topbarbg; - color: $linkcol; + background: colors.$inputbg; + border: 1px solid colors.$topbarbg; + color: colors.$linkcol; } } th.postblock, table.mgmt-table tr:first-of-type th { - background: $blockcol; - border: 1px solid $blockborder; + background: colors.$blockcol; + border: 1px solid colors.$blockborder; border-radius: 5px; text-align: left; } @@ -43,13 +43,13 @@ th.postblock, table.mgmt-table tr:first-of-type th { div.reply, div.postprev, div.inlinepostprev { - background: $inputbg; - border: 1px solid $replyborder; + background: colors.$inputbg; + border: 1px solid colors.$replyborder; border-radius: 5px; } span.postername { - color: $headercol; + color: colors.$headercol; font-weight: bold; a { text-decoration: underline; @@ -57,18 +57,18 @@ span.postername { } span.tripcode { - color: $headercol; + color: colors.$headercol; } .dropdown-menu { - background: $topbarbg!important; + background: colors.$topbarbg!important; b { color: black; } } span.subject { - color: $subjectcol; + color: colors.$subjectcol; } table#pages, table#pages * { diff --git a/frontend/sass/darkbunker.scss b/frontend/sass/darkbunker.scss index 97476b96..6fac77e7 100644 --- a/frontend/sass/darkbunker.scss +++ b/frontend/sass/darkbunker.scss @@ -1,59 +1,59 @@ -@import './darkbunker/img'; -@import './darkbunker/vars'; -@import './util'; +@use 'darkbunker/img'; +@use 'darkbunker/vars'; +@use 'util'; body { - background: $bgcol; - color: $txtcol; + background: vars.$bgcol; + color: vars.$txtcol; font-family: arial,helvetica,sans-serif; font-size: 10pt; } div#topbar { - background: $topbarcol; - border-bottom: 2px solid $linkcol; + background: vars.$topbarcol; + border-bottom: 2px solid vars.$linkcol; * { padding: 0px; } } hr { - color: $bordercol; + color: vars.$bordercol; } a { - color: $linkcol; + color: vars.$linkcol; text-decoration: none; } a:hover { background: inherit; - color: $linklight; + color: vars.$linklight; } header { h1, div#board-subtitle { font-family: tahoma; - color: $headercol; + color: vars.$headercol; } } th, div.reply { - background: $gridcol; - border: 1px solid $bordercol; + background: vars.$gridcol; + border: 1px solid vars.$bordercol; } div.section-block { border: 1px solid white; div.section-title-block { - background: $blocktitle; + background: vars.$blocktitle; border: 1px solid white; font-weight: bold; } } div#frontpage div.section-block:first-child { - background: $gridcol; + background: vars.$gridcol; // border: none; } \ No newline at end of file diff --git a/frontend/sass/darkbunker/_img.scss b/frontend/sass/darkbunker/_img.scss index 52b445f7..0228b59e 100644 --- a/frontend/sass/darkbunker/_img.scss +++ b/frontend/sass/darkbunker/_img.scss @@ -1,23 +1,23 @@ -@import './vars'; +@use 'vars'; div#content, div#qr-box { - color: $txtcol; - border: 1px double $inputshadow; + color: vars.$txtcol; + border: 1px double vars.$inputshadow; border-radius: 5px; textarea, input:not([type="file"]):not([type="checkbox"]), [type="submit"], select { - color: $txtcol; - background: $gridcol; - border: 1px solid $inputshadow; + color: vars.$txtcol; + background: vars.$gridcol; + border: 1px solid vars.$inputshadow; border-radius: 5px; } } div#qr-box { - background: $gridcol; + background: vars.$gridcol; } form#postform, form#qrpostform { @@ -29,5 +29,5 @@ form#postform, form#qrpostform { div#staffmenu, div#watchermenu { background: black; - border: 1px solid $txtcol; + border: 1px solid vars.$txtcol; } diff --git a/frontend/sass/global.scss b/frontend/sass/global.scss index cebaf5a1..861b9c10 100644 --- a/frontend/sass/global.scss +++ b/frontend/sass/global.scss @@ -1,11 +1,11 @@ -@import 'global/img'; -@import 'global/front'; -@import 'global/manage'; -@import 'global/lightbox'; -@import 'global/qr'; -@import "global/watcher"; -@import 'global/bans'; -@import 'global/animations'; +@use 'global/img'; +@use 'global/front'; +@use 'global/manage'; +@use 'global/lightbox'; +@use 'global/qr'; +@use "global/watcher"; +@use 'global/bans'; +@use 'global/animations'; .increase-line-height { header, .post, .reply { diff --git a/frontend/sass/global/_img.scss b/frontend/sass/global/_img.scss index 694f1010..1d3d0588 100755 --- a/frontend/sass/global/_img.scss +++ b/frontend/sass/global/_img.scss @@ -1,4 +1,4 @@ -@import 'animations'; +@use 'animations'; #boardmenu-bottom { margin-top: 16px; diff --git a/frontend/sass/global/_lightbox.scss b/frontend/sass/global/_lightbox.scss index ea2a12aa..6b797837 100644 --- a/frontend/sass/global/_lightbox.scss +++ b/frontend/sass/global/_lightbox.scss @@ -1,4 +1,4 @@ -@import '../util'; +@use '../util'; .lightbox { background:#CDCDCD; @@ -15,8 +15,7 @@ } .lightbox * { - // box-shadow: 0px 0px 0px 0px #000000; - @include shadow-filter(0px 0px 0px #000); + @include util.shadow-filter(0px 0px 0px #000); color:#000000; } @@ -44,8 +43,6 @@ } .lightbox-title { - // font-size:42px; - // font-weight:700; text-align:center; margin-top: 0px; } @@ -54,7 +51,6 @@ color: #000!important; float: right; font-size: inherit; - // font-weight: 700; } .lightbox-x:hover,.lightbox-x:active { @@ -70,14 +66,14 @@ .lightbox input[type=password], .lightbox input[type=file], .lightbox textarea { - background:#FFF; - border:1px solid #000; - color:#000; + background: #FFF; + border: 1px solid #000; + color: #000; } .lightbox textarea#sql-statement { - width:95%; - height:300px; + width: 95%; + height: 300px; margin-left: 0px; clear: both; background: #FFF; @@ -92,8 +88,7 @@ border: 1px solid #000; border-radius: 0px; background: #777; - // box-shadow: 0px 0px 0px 0px #000000; - @include shadow-filter(0px 0px 0px #000); + @include util.shadow-filter(0px 0px 0px #000); } #settings-container table textarea { diff --git a/frontend/sass/global/_qr.scss b/frontend/sass/global/_qr.scss index 95329d02..0149495b 100644 --- a/frontend/sass/global/_qr.scss +++ b/frontend/sass/global/_qr.scss @@ -1,8 +1,8 @@ -@import '../util'; +@use '../util'; div#qr-box { padding:1px; - @include box-sizing(border-box); + @include util.box-sizing(border-box); min-width: 300px; background:lightgray; border: 1px solid #000; @@ -13,7 +13,7 @@ div#qr-box { background: #FFF; color: #000; width:100%; - @include box-sizing(border-box); + @include util.box-sizing(border-box); } input[type=file] { background: lightgray; diff --git a/frontend/sass/photon.scss b/frontend/sass/photon.scss index 18a16682..ddeed809 100644 --- a/frontend/sass/photon.scss +++ b/frontend/sass/photon.scss @@ -1,10 +1,10 @@ -@import 'photon/colors'; -@import 'photon/img'; -@import 'util'; +@use 'photon/colors'; +@use 'photon/img'; +@use 'util'; body { - background: $bgcol; - font: 15px $font; + background: colors.$bgcol; + font: 15px colors.$font; } a { @@ -18,25 +18,25 @@ div#topbar, div.dropdown-menu { } div#topbar { - @include shadow-filter(0px 2px 2px $shadowcol); + @include util.shadow-filter(0px 2px 2px colors.$shadowcol); } .dropdown-menu { color: #FFF; background: #000!important; - @include shadow-filter(2px 2px 3px $shadowcol); + @include util.shadow-filter(2px 2px 3px colors.$shadowcol); z-index: 0; div:hover { - background: $dropdowncol; + background: colors.$dropdowncol; } } header, div#top-pane, a { - color: $linkcol; + color: colors.$linkcol; } #site-title, #board-title { font-weight: 800; } -@include upload-box(#aaa, #444, #666); \ No newline at end of file +@include util.upload-box(#aaa, #444, #666); \ No newline at end of file diff --git a/frontend/sass/photon/_img.scss b/frontend/sass/photon/_img.scss index 3e4ffe9d..486077e8 100644 --- a/frontend/sass/photon/_img.scss +++ b/frontend/sass/photon/_img.scss @@ -1,25 +1,25 @@ -@import '../util'; -@import 'colors'; +@use '../util'; +@use 'colors'; div.reply, div.postprev, div.inlinepostprev, th.postblock { - background: $replycol; - border: 1px solid $replyborder; - @include border-radius(5px); + background: colors.$replycol; + border: 1px solid colors.$replyborder; + @include util.border-radius(5px); } span.subject { - color: $subjectcol; + color: colors.$subjectcol; } div.section-title-block { - background: $replyborder; + background: colors.$replyborder; } div.section-block { - @include border-radius(5px); - background: $sectionbg; - border: 1px solid $sectionborder; + @include util.border-radius(5px); + background: colors.$sectionbg; + border: 1px solid colors.$sectionborder; } \ No newline at end of file diff --git a/frontend/sass/pipes.scss b/frontend/sass/pipes.scss index 6bf73e38..1e37e4de 100644 --- a/frontend/sass/pipes.scss +++ b/frontend/sass/pipes.scss @@ -1,37 +1,37 @@ -@import 'pipes/colors'; -@import 'pipes/front'; -@import 'pipes/manage'; -@import 'pipes/img'; -@import 'util'; +@use 'pipes/colors'; +@use 'pipes/front'; +@use 'pipes/manage'; +@use 'pipes/img'; +@use 'util'; * { outline: none; } body { - background: $bgcol; + background: colors.$bgcol; background-image: url(res/pipes_bg.png); color: #d8d0b9; - font: $font; + font: colors.$font; } header h1, #site-title { - color: $headercol; + color: colors.$headercol; } a { - color: $linkcol; - font: $font; + color: colors.$linkcol; + font: colors.$font; text-decoration: none; } a:hover { background: inherit; - color: $linklight; + color: colors.$linklight; } a.topbar-item:hover { - background: $bglight; + background: colors.$bglight; } footer, footer * { @@ -39,35 +39,35 @@ footer, footer * { } div#topbar { - background: $topbarcol; + background: colors.$topbarcol; // @include box-shadow(0px 2px 2px 3px $shadowcol); - @include shadow-filter(0px 4px 2px $shadowcol); + @include util.shadow-filter(0px 4px 2px colors.$shadowcol); li:hover { - background: $bglight; + background: colors.$bglight; } } .dropdown-menu { - background: $topbarcol!important; - @include shadow-filter(2px 2px 3px $shadowcol); + background: colors.$topbarcol!important; + @include util.shadow-filter(2px 2px 3px colors.$shadowcol); z-index: 0; div:hover { - background: $bglight; + background: colors.$bglight; } } .ui-tabs-tab { - background: $topbarcol; - border: 1px solid $bglight; + background: colors.$topbarcol; + border: 1px solid colors.$bglight; } .ui-tabs-active { - background: $bgcol; + background: colors.$bgcol; } .ui-tabs-panel { - background: $bgcol; - border: 1px solid $topbarcol; + background: colors.$bgcol; + border: 1px solid colors.$topbarcol; padding: 8px; } \ No newline at end of file diff --git a/frontend/sass/pipes/_front.scss b/frontend/sass/pipes/_front.scss index ced579ee..911345d5 100644 --- a/frontend/sass/pipes/_front.scss +++ b/frontend/sass/pipes/_front.scss @@ -1,8 +1,8 @@ -@import 'colors'; -@import '../util'; +@use 'colors'; +@use '../util'; .dropdown-button:hover { - background: $bglight; + background: colors.$bglight; } .section-body { @@ -10,27 +10,27 @@ } .section-title-block { - background: $topbarcol; + background: colors.$topbarcol; border-radius: 4px 4px 0px 0px; } .tab { - background: $bglight; - border: 1px solid $bgcol; + background: colors.$bglight; + border: 1px solid colors.$bgcol; } #current-tab { - background: $bgcol; + background: colors.$bgcol; } div#recent-posts-header { // @include box-shadow(0px 2px 2px 3px $shadowcol); - @include shadow-filter(0px 2px 3px $shadowcol); + @include util.shadow-filter(0px 2px 3px colors.$shadowcol); margin-bottom: 8px; padding: 4px 8px 4px 8px; } .postblock { - background: $bgcol; + background: colors.$bgcol; font-weight: 700; } diff --git a/frontend/sass/pipes/_img.scss b/frontend/sass/pipes/_img.scss index 289897cb..1b79a34b 100644 --- a/frontend/sass/pipes/_img.scss +++ b/frontend/sass/pipes/_img.scss @@ -1,8 +1,8 @@ -@import '_colors'; -@import '../util'; +@use '_colors'; +@use '../util'; .dropdown-button:hover { - background: $bglight; + background: colors.$bglight; } img.thumbnail { @@ -12,13 +12,13 @@ img.thumbnail { } .reply, .inlinepostprev, .postprev { - background: $postblock; - border: 1px solid $postblockcol; + background: colors.$postblock; + border: 1px solid colors.$postblockcol; } .postblock, table.mgmt-table tr:first-of-type th { - background: $postblock; + background: colors.$postblock; font-weight: 700; } @@ -45,8 +45,8 @@ div#content { textarea, select#changepage, select.post-actions { - background: $postblock; - border: 1px solid $postblockcol; + background: colors.$postblock; + border: 1px solid colors.$postblockcol; color: #FFF; } } diff --git a/frontend/sass/pipes/_manage.scss b/frontend/sass/pipes/_manage.scss index 1b9c4fae..3a030b50 100644 --- a/frontend/sass/pipes/_manage.scss +++ b/frontend/sass/pipes/_manage.scss @@ -1,13 +1,13 @@ -@import 'colors'; -@import '../util'; +@use 'colors'; +@use '../util'; .loginbox input { height: 20%; } .manage-header { - background: $topbarcol; + background: colors.$topbarcol; // @include box-shadow(2px 2px 3px 4px $shadowcol); - @include shadow-filter(4px 4px 4px $shadowcol); + @include util.shadow-filter(4px 4px 4px colors.$shadowcol); border-radius: 8px; } \ No newline at end of file diff --git a/frontend/sass/win9x.scss b/frontend/sass/win9x.scss index 127c3ad7..93e9879a 100755 --- a/frontend/sass/win9x.scss +++ b/frontend/sass/win9x.scss @@ -1,27 +1,27 @@ -@import 'win9x/vars'; +@use 'win9x/vars'; @font-face { - @include mssans-font(400, normal); + @include vars.mssans-font(400, normal); } @font-face { - @include mssans-font(700, normal); + @include vars.mssans-font(700, normal); } body { - font-size: $fontsize; + font-size: vars.$fontsize; color: white; - background: $bgcol; + background: vars.$bgcol; } a { - color: $txtcol; + color: vars.$txtcol; text-decoration: none; } input { - color: $txtcol; - font-size: $fontsize!important; + color: vars.$txtcol; + font-size: vars.$fontsize!important; font-family: "Pixelated MS Sans Serif", Arial; outline: none; } @@ -30,7 +30,7 @@ input { div#topbar { height: 26px; background: silver!important; - box-shadow: $bar-shadow; + box-shadow: vars.$bar-shadow; box-sizing: border-box; border: none!important; border-radius: 0!important; @@ -39,9 +39,9 @@ div#topbar { } a[href="/"] { font-family: "Pixelated MS Sans Serif", Arial; - font-size: $fontsize; + font-size: vars.$fontsize; content: "Start"!important; - background: $start_path!important; + background: vars.$start_path!important; background-repeat: no-repeat!important; background-position-x: 3px!important; background-position-y: 4px!important; @@ -54,12 +54,12 @@ div#topbar { div#topbar, div#topbar a { background: silver!important; - box-shadow: $bar-shadow; + box-shadow: vars.$bar-shadow; box-sizing: border-box; border: none!important; border-radius: 0!important; height:24px; - color: $txtcol!important; + color: vars.$txtcol!important; font-family: "Pixelated MS Sans Serif", Arial; font-size: 11px; cursor: default; @@ -70,10 +70,10 @@ div#content { input[type=button], input[role=pushbutton], input[type=submit] { - color: $txtcol!important; + color: vars.$txtcol!important; /* font-size: 12px!important; */ background: silver!important; - box-shadow: $bar-shadow; + box-shadow: vars.$bar-shadow; box-sizing: border-box; border: none!important; border-radius: 0!important; @@ -95,7 +95,7 @@ div#qr-title { background: linear-gradient(90deg,navy,#1084d0); a { font-family: Arial, Helvetica, sans-serif; - box-shadow: $bar-shadow; + box-shadow: vars.$bar-shadow; min-width: 16px; min-height: 14px; display: block; @@ -105,7 +105,7 @@ div#qr-title { background-position: top 3px left 40px!important; } a:hover { - color:$txtcol; + color:vars.$txtcol; } span#qr-buttons * { display: inline-block; @@ -125,7 +125,7 @@ div#qr-box { box-shadow: inset -1px -1px #fff,inset 1px 1px grey,inset -2px -2px #dfdfdf,inset 2px 2px #0a0a0a; margin: 0; font-family: "Pixelated MS Sans Serif",Arial; - color: $txtcol; + color: vars.$txtcol; appearance: none; } } @@ -140,7 +140,7 @@ input[type=text] { } */ a.boardlistactive { - color: $txtcol!important; + color: vars.$txtcol!important; padding: 6px 3px!important; outline: 1px dotted #000; outline-offset: -4px; @@ -148,7 +148,7 @@ a.boardlistactive { h1, h2, div.subtitle, a#qrDisplayButton { - color: $txtcol; + color: vars.$txtcol; font-family: Arial, sans-serif; } @@ -172,7 +172,7 @@ div#content select { } div.pages { - background: $bgcol; + background: vars.$bgcol; border: none; color: white; /* border-right: black; diff --git a/frontend/sass/yotsuba.scss b/frontend/sass/yotsuba.scss index a0a11316..d6486eb4 100644 --- a/frontend/sass/yotsuba.scss +++ b/frontend/sass/yotsuba.scss @@ -1,23 +1,23 @@ -@import 'global/colors'; -@import 'yotsuba/colors'; -@import 'yotsubacommon'; +@use 'global/colors' as global-colors; +@use 'yotsuba/colors' as yotsuba-colors; +@use 'yotsubacommon'; -@include yotsuba( +@include yotsubacommon.yotsuba( 'res/yotsuba_bg.png', - $bgcol, /* $bodybg */ + yotsuba-colors.$bgcol, /* $bodybg */ maroon, /* $bodycol */ - $topbarbg, - $topbarborder, - $headercol, - $postblockbg, - $postblockborder, - $postblockborder, /* $sectiontitlecol */ + yotsuba-colors.$topbarbg, + yotsuba-colors.$topbarborder, + global-colors.$headercol, + yotsuba-colors.$postblockbg, + yotsuba-colors.$postblockborder, + yotsuba-colors.$postblockborder, /* $sectiontitlecol */ #fca, /* $sectiontitlebg */ - $borderbotright, + yotsuba-colors.$borderbotright, maroon, /* $linkcol */ #D9BfB7, /* $hrcol */ #CC1105, /* $subjectcol */ - $namecol, - $replybg, + global-colors.$namecol, + yotsuba-colors.$replybg, navy /* $postlinkcol */ ); diff --git a/frontend/sass/yotsubab.scss b/frontend/sass/yotsubab.scss index 26990949..8f8c0297 100644 --- a/frontend/sass/yotsubab.scss +++ b/frontend/sass/yotsubab.scss @@ -1,23 +1,23 @@ -@import 'global/colors'; -@import 'yotsubab/colors'; -@import 'yotsubacommon'; +@use 'global/colors' as global-colors; +@use 'yotsubab/colors' as yotsubab-colors; +@use 'yotsubacommon'; -@include yotsuba( +@include yotsubacommon.yotsuba( 'res/yotsubab_bg.png', - $bgcol, /* $bodybg */ + yotsubab-colors.$bgcol, /* $bodybg */ #000, /* $bodycol */ #D6DAF0, /* $topbarbg */ - $topbarborder, - $headercol, - $postblockbg, - $postblockborder, + yotsubab-colors.$topbarborder, + global-colors.$headercol, + yotsubab-colors.$postblockbg, + yotsubab-colors.$postblockborder, #000, /* $sectiontitlecol */ - $postblockbg, /* $sectiontitlebg */ - $borderbotright, + yotsubab-colors.$postblockbg, /* $sectiontitlebg */ + yotsubab-colors.$borderbotright, #34345C, /* $linkcol */ #B7C5D9, /* $hrcol */ #0F0C5D, /* $subjectcol */ - $namecol, - $replybg, + global-colors.$namecol, + yotsubab-colors.$replybg, navy /* $postlinkcol */ ); diff --git a/html/css/global.css b/html/css/global.css index 0d521fd2..dfb85507 100644 --- a/html/css/global.css +++ b/html/css/global.css @@ -641,26 +641,6 @@ img#banpage-image { margin: 4px 8px 8px 4px; } -@keyframes slideopen { - from { - transform: scale(1, 0); - transform-origin: top center; - } - to { - transform: scale(1, 1); - transform-origin: top center; - } -} -@keyframes slideclose { - from { - transform: scale(1, 1); - transform-origin: top center; - } - to { - transform: scale(1, 0); - transform-origin: top center; - } -} .increase-line-height header, .increase-line-height .post, .increase-line-height .reply { line-height: 1.5; } From 1339f58a66ced7fa3d2c68792aa1a2d2b62a3d7b Mon Sep 17 00:00:00 2001 From: Eggbertx Date: Sat, 22 Feb 2025 11:35:03 -0800 Subject: [PATCH 122/122] Add defer to gochan.js script tag, add option to open external links in new tab --- frontend/ts/gochan.ts | 3 ++- frontend/ts/settings.ts | 17 +++++++++++++++-- templates/page_header.html | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/frontend/ts/gochan.ts b/frontend/ts/gochan.ts index 88508774..340b0a83 100755 --- a/frontend/ts/gochan.ts +++ b/frontend/ts/gochan.ts @@ -4,7 +4,7 @@ import "./vars"; import "./cookies"; import "./notifications"; import { setPageBanner } from "./dom/banners"; -import { setCustomCSS, setCustomJS, setTheme } from "./settings"; +import { setCustomCSS, setCustomJS, setTheme, updateExternalLinks } from "./settings"; import { handleKeydown } from "./boardevents"; import { initStaff, createStaffMenu, addStaffThreadOptions } from "./management/manage"; import { getPageThread } from "./postinfo"; @@ -57,6 +57,7 @@ $(() => { }); $(document).on("keydown", handleKeydown); initFlags(); + updateExternalLinks(); setCustomCSS(); setCustomJS(); }); diff --git a/frontend/ts/settings.ts b/frontend/ts/settings.ts index b51d66c5..9667a3a4 100755 --- a/frontend/ts/settings.ts +++ b/frontend/ts/settings.ts @@ -217,6 +217,19 @@ function setLineHeight() { } } +export function updateExternalLinks(post?: JQuery) { + const $src = post ?? $(".post-text"); + const extPostLinks = $src.find("a:not(.postref)"); + const newTab = getBooleanStorageVal("extlinksnewtab", true); + for(const link of extPostLinks) { + if(link.hostname !== location.hostname) { + link.target = newTab?"_blank":"_self"; + } else { + link.target = "_self"; + } + } +} + /** * executes the custom JavaScript set in the settings */ @@ -266,6 +279,7 @@ $(() => { if(getBooleanStorageVal("useqr", true)) initQR(); else closeQR(); })); + settings.set("extlinksnewtab", new BooleanSetting("extlinksnewtab", "Open external links in new tab", true, updateExternalLinks)); settings.set("persistentqr", new BooleanSetting("persistentqr", "Persistent Quick Reply", false)); settings.set("watcherseconds", new NumberSetting("watcherseconds", "Check watched threads every # seconds", 15, { min: 2 @@ -276,6 +290,5 @@ $(() => { settings.set("customjs", new TextSetting("customjs", "Custom JavaScript", "")); settings.set("customcss", new TextSetting("customcss", "Custom CSS", "", setCustomCSS)); - if($settingsButton === null) - $settingsButton = new TopBarButton("Settings", createLightbox, {before: "a#watcher"}); + $settingsButton ??= new TopBarButton("Settings", createLightbox, {before: "a#watcher"}); }); \ No newline at end of file diff --git a/templates/page_header.html b/templates/page_header.html index 2cf35247..fd58bf86 100644 --- a/templates/page_header.html +++ b/templates/page_header.html @@ -27,7 +27,7 @@ {{- end -}} - + {{template "topbar" .}}